From 895d5fea41177e444c18f4fdc820fffa5f67d5bf Mon Sep 17 00:00:00 2001 From: Rasmus Dahlberg Date: Sat, 9 Dec 2023 17:08:45 +0100 Subject: Add drafty skeleton --- README.md | 131 +++++++++++--------- contrib/lets-encrypt.sh | 3 + docs/architecture.md | 3 + docs/server.md | 8 ++ docs/state.md | 9 ++ go.mod | 16 ++- go.sum | 29 +++++ internal/manager/helpers.go | 52 ++++++++ internal/manager/manager.go | 94 +++++++++++++++ internal/merkle/TODO | 1 + internal/merkle/compact.go | 115 ++++++++++++++++++ internal/merkle/merkle.go | 271 +++++++++++++++++++++++++++++++++++++++++ internal/options/options.go | 97 +++++++++++++++ main.go | 105 +++++++++++++++- pkg/monitor/chunks.go | 88 ++++++++++++++ pkg/monitor/errors.go | 41 +++++++ pkg/monitor/matcher.go | 90 ++++++++++++++ pkg/monitor/messages.go | 40 +++++++ pkg/monitor/monitor.go | 286 ++++++++++++++++++++++++++++++++++++++++++++ pkg/server/errors.go | 1 + pkg/server/messages.go | 4 + pkg/server/server.go | 106 ++++++++++++++++ 22 files changed, 1530 insertions(+), 60 deletions(-) create mode 100644 contrib/lets-encrypt.sh create mode 100644 docs/architecture.md create mode 100644 docs/server.md create mode 100644 docs/state.md create mode 100644 go.sum create mode 100644 internal/manager/helpers.go create mode 100644 internal/manager/manager.go create mode 100644 internal/merkle/TODO create mode 100644 internal/merkle/compact.go create mode 100644 internal/merkle/merkle.go create mode 100644 internal/options/options.go create mode 100644 pkg/monitor/chunks.go create mode 100644 pkg/monitor/errors.go create mode 100644 pkg/monitor/matcher.go create mode 100644 pkg/monitor/messages.go create mode 100644 pkg/monitor/monitor.go create mode 100644 pkg/server/errors.go create mode 100644 pkg/server/messages.go create mode 100644 pkg/server/server.go diff --git a/README.md b/README.md index d510068..44b2da0 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ An implementation of a silent Certificate Transparency (CT) monitor. +**Status:** work in progress, please do not use for anything serious. + ## How it works Each node that issues TLS certificates submit them to a self-hosted monitor. @@ -12,101 +14,116 @@ An overview is shown in the figure below. XXX: ASCII figure. -To establish a secure connection to the monitor, a Tor onion service is assumed. -This means that all nodes and the monitor need to have Tor installed locally. - -The use of a Tor onion service also allows for NAT punching. In other words, it -is possible to run the monitor without a publicly reachable IP address. +Each node establishes a secure channel to the monitor. Secure means end-to-end +encrypted communication where the node can authenticate the monitor. For the +monitor to authenticate each connecting node, HTTP Basic Authentication is used. -For the monitor to authenticate each node, HTTP Basic Authentication is used. +On a single-node system, the secure channel can be replaced by a UNIX socket. ## Setting and threat model - 1. It is not always possible for nodes to reach the monitor. For example, the - monitor may be running on a workstation only powered on during work hours. - 2. An administrator notices alerts that `silent-ctmoon` outputs on stdout. - The integration with email, dashboards, or similar is out of scope. - 3. The platform running the monitor is not compromised by the attacker. - 4. The platforms hosting TLS sites start in good states but may be compromised - by the attacker sometime in the future. Detection of certificate - mis-issuance is then out of scope for the domains the compromised node - issued certificates for. It is in-scope to not let such a compromise - affect detection of mis-issued certificates with other nodes' domains. - 5. Mis-issued certificates will only be used for MitM attacks against users + 1. Mis-issued certificates will only be used for MitM attacks against users that connect from a set of fixed IP addresses. It is assumed that the party who can detect certificate mis-issuance will never be targeted. This is why each node needs to establish a secure channel with the monitor. + 2. The nodes issuing TLS certificates start in good states but may be + compromised sometime in the future. Detection of certificate mis-issuance + is then out of scope for all domains that the compromised node issued + certificates for. It is in-scope to not let such a compromise affect + detection of mis-issued certificates for other nodes' domains though. + 3. The platform running the monitor is not compromised by the attacker. + 4. An administrator notices alerts that the monitor outputs on stdout. The + integration with email, dashboards, or similar is out of scope. + 5. It is not always possible for nodes to reach the monitor. For example, the + monitor may be running on a workstation only powered on during work hours. ## Install -Install Tor on all platforms. On Debian: +Install the `silent-ct` software using Go's toolchain: - # apt install tor + $ go install rgdd.se/silent-ct@latest -Install the silent-ct software using Go's toolchain: +If the monitor is not running on the same system as a single node, it is up to +you to configure an end-to-end encrypted channel where the monitor is +authenticated. The remainder of this README assumed use of a Tor onion service. +Note that the monitor will then punch any NAT and thus run without a public IP. - $ go install rgdd.se/silent-ct/cmd/silent-ctnode@latest - $ go install rgdd.se/silent-ct/cmd/silent-ctmoon@latest +## Configure the monitor -`silent-ctnode` is only used on nodes that issue TLS certificates. +Create a configuration file specifying which nodes can issue what certificates. +You will also need to specify which domains the monitor should be looking for. -`silent-ctmoon` is only used on the system that runs the monitor. + $ cat config + { + "monitor": [ + { + "wildcard": "example.org", + "excludes": [ + "test" + ] + } + ], + "nodes": [ + { + "name": "NODE_NAME", + "secret": "NODE_SECRET", + "issues": [ + "example.org", + "www.example.org" + ] + } + ] + } + +Here, the monitor is looking for `example.com` and all its subdomains expect for +anything relating to `test.example.com`. There is a single node that is allowed +to issue certificates with the domain names `example.org` and `www.example.org`. + +## Start the monitor + +[Setup an onion service][] that routes through the below UNIX socket. + +Start the monitor: + + $ mkdir state + $ silent-ctmoon -c config.json -s state -l silent-ctmoon.sock + +The intended way to exit is by sending the SIGINT or SIGTERM signals. + +[Setup an onion service]: xxx ## Node setup -What makes `silent-ctmoon` _silent_ is that all nodes report legitimately issued +What makes `silent-ct` _silent_ is that all nodes report legitimately issued certificates over a secure channel. For each such legitimate certificate, run: - $ torify silent-ctnode -n NODE_NAME -s NODE_SECRET -o ONION_ADDR -c CHAIN_PATH + $ torify curl ... `NODE_NAME` is an arbitrary node name used for authentication. `NODE_SECRET` is an arbitrary node secret used for authentication. -`ONION_ADDR` is the onion address where `silent-ctmoon` listens for requests. +`ONION_ADDR` is the onion address where `silent-ct` listens for requests. `CHAIN_PATH` is the path to a certificate chain that was issued legitimately. It is safe to repeatedly submit the same certificate chain to `silent-ctmoon`. You will need to add a cron job (or similar) that periodically submits the -issued certificates to `silent-ctmoon`. See example that works for `certbot` +issued certificates to `silent-ct`. See example that works for `certbot` issuing certificates with Let's Encrypt in the [contrib/](./contrib) directory. -## Configure the monitor - -Create a configuration file specifying which nodes can issue what certificates. - -Example: - - $ cat config - NODE_A aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa example.org www.example.org - NODE_B bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb git.example.org - -The first column specifies an arbitrary node name. - -The second column specifies a node secret used for mutual authentication. - -The following columns specify domain names the node issues certificates for. - -## Start the monitor - -[Configure an onion service][] that routes through the below UNIX socket. - -Start the monitor: - - $ silent-ctmoon -c config -s state.db -l silent-ctmoon.sock - -Stop gracefully by sending the SIGINT signal. - -[Configure an onion service]: xxx - ## Read monitoring alerts All output is formatted as follows: : -Only two keywords are expected by default: `alert` and `recovered`. +Only two keywords are expected by default: `fatal` and `alert`. + +`fatal`: fatal errors were encountered, `silent-ct` is no longer running. + +`alert`: something that you want to know about has happened, such as a +certificate appearing in the logs that none of the trusted nodes submitted. ## Contact diff --git a/contrib/lets-encrypt.sh b/contrib/lets-encrypt.sh new file mode 100644 index 0000000..a1fbd02 --- /dev/null +++ b/contrib/lets-encrypt.sh @@ -0,0 +1,3 @@ +#/bin/sh + +echo "To be added" diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..8f562cb --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,3 @@ +# Architecture + +docdoc diff --git a/docs/server.md b/docs/server.md new file mode 100644 index 0000000..51a5ec4 --- /dev/null +++ b/docs/server.md @@ -0,0 +1,8 @@ +# Server API + +docdoc + +## Authentication +## Endpoints +### get-status +### add-chain diff --git a/docs/state.md b/docs/state.md new file mode 100644 index 0000000..fbb965d --- /dev/null +++ b/docs/state.md @@ -0,0 +1,9 @@ +# State + +All state is managed by the silent-ct [manager](../internal/manager). All such +state is persisted to disk in a common directory as a collection of files. Not +having any database simplifies setup and manual debugging, should it be needed. + +## Structure of the state directory + +docdoc diff --git a/go.mod b/go.mod index da7e70a..61e0b57 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,17 @@ module rgdd.se/silent-ct -go 1.21.3 +go 1.19 + +require ( + github.com/go-logr/logr v1.2.4 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/certificate-transparency-go v1.1.7 // indirect + github.com/google/trillian v1.5.3 // indirect + gitlab.torproject.org/rgdd/ct v0.0.0-20230508072727-1d1808eac7db // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/grpc v1.59.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + k8s.io/klog/v2 v2.100.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b9acc1d --- /dev/null +++ b/go.sum @@ -0,0 +1,29 @@ +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/certificate-transparency-go v1.1.7 h1:IASD+NtgSTJLPdzkthwvAG1ZVbF2WtFg4IvoA68XGSw= +github.com/google/certificate-transparency-go v1.1.7/go.mod h1:FSSBo8fyMVgqptbfF6j5p/XNdgQftAhSmXcIxV9iphE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/trillian v1.5.3 h1:3ioA5p09qz+U9/t2riklZtaQdZclaStp0/eQNfewNRg= +github.com/google/trillian v1.5.3/go.mod h1:p4tcg7eBr7aT6DxrAoILpc3uXNfcuAvZSnQKonVg+Eo= +gitlab.torproject.org/rgdd/ct v0.0.0-20230508072727-1d1808eac7db h1:wR2Fc+fRDBkpabB1og472+HCoNtE4TJEuAHbSG4EIEs= +gitlab.torproject.org/rgdd/ct v0.0.0-20230508072727-1d1808eac7db/go.mod h1:dkEqBVulcsefxw3k5CX53bw88KUkTYcTRSJhMlj1Veg= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= diff --git a/internal/manager/helpers.go b/internal/manager/helpers.go new file mode 100644 index 0000000..a9a2158 --- /dev/null +++ b/internal/manager/helpers.go @@ -0,0 +1,52 @@ +package manager + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" + + ct "github.com/google/certificate-transparency-go" + "gitlab.torproject.org/rgdd/ct/pkg/metadata" + "rgdd.se/silent-ct/pkg/monitor" +) + +func selectLogs(m metadata.Metadata) []monitor.MessageLogConfig { + var logs []monitor.MessageLogConfig + for _, operator := range m.Operators { + for _, log := range operator.Logs { + if log.State == nil { + continue // ignore logs without a state (should not happen) + } + if log.State.Name == metadata.LogStatePending { + continue // log is not yet relevant + } + if log.State.Name == metadata.LogStateRetired { + continue // log is not expected to be reachable + } + if log.State.Name == metadata.LogStateRejected { + continue // log is not expected to be reachable + } + + // FIXME: remove me instead of hard coding Argon 2024 + id, _ := log.Key.ID() + got := fmt.Sprintf("%s", base64.StdEncoding.EncodeToString(id[:])) + want := "7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=" + if got != want { + continue + } + + logs = append(logs, monitor.MessageLogConfig{ + Metadata: log, + State: monitor.MonitorState{ + LogState: monitor.LogState{ct.SignedTreeHead{ + SHA256RootHash: [sha256.Size]byte{47, 66, 110, 15, 246, 154, 8, 100, 150, 140, 206, 208, 17, 57, 112, 116, 210, 3, 19, 55, 46, 63, 209, 12, 234, 130, 225, 124, 237, 2, 64, 228}, + TreeSize: 610650601, + Timestamp: 1702108968538, + }}, + NextIndex: 388452203, + }, + }) + } + } + return logs +} diff --git a/internal/manager/manager.go b/internal/manager/manager.go new file mode 100644 index 0000000..2210c9b --- /dev/null +++ b/internal/manager/manager.go @@ -0,0 +1,94 @@ +package manager + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "gitlab.torproject.org/rgdd/ct/pkg/metadata" + "rgdd.se/silent-ct/pkg/monitor" + "rgdd.se/silent-ct/pkg/server" +) + +const ( + DefaultStateDir = "/home/rgdd/.local/share/silent-ct" // FIXME + DefaultMetadataRefreshInterval = 1 * time.Hour +) + +type Config struct { + StateDir string + Nodes server.Nodes + + MetadataRefreshInterval time.Duration +} + +type Manager struct { + Config +} + +func New(cfg Config) (Manager, error) { + if cfg.StateDir == "" { + cfg.StateDir = DefaultStateDir + } + if cfg.MetadataRefreshInterval == 0 { + cfg.MetadataRefreshInterval = DefaultMetadataRefreshInterval + } + return Manager{Config: cfg}, nil +} + +func (mgr *Manager) Run(ctx context.Context, + serverCh chan server.MessageNodeSubmission, + monitorCh chan monitor.MessageLogProgress, + configCh chan []monitor.MessageLogConfig, + errorCh chan error) error { + + md, err := mgr.metadataRead() + if err != nil { + return fmt.Errorf("read metadata: %v\n", err) + } + configCh <- selectLogs(md) + + ticker := time.NewTicker(mgr.MetadataRefreshInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + mu, err := mgr.metadataUpdate(ctx, md) + if err != nil { + continue + } + if mu.Version.Major <= md.Version.Major { + continue + } + md = mu + configCh <- selectLogs(md) + case ev := <-monitorCh: + fmt.Printf("DEBUG: received event from monitor with %d matches\n", len(ev.Matches)) + case ev := <-serverCh: + fmt.Printf("DEBUG: received event from server\n: %v", ev) + case err := <-errorCh: + fmt.Printf("DEBUG: received error: %v\n", err) + } + } +} + +func (mgr *Manager) metadataRead() (metadata.Metadata, error) { + b, err := os.ReadFile(mgr.StateDir + "/metadata.json") + if err != nil { + return metadata.Metadata{}, err + } + var md metadata.Metadata + if err := json.Unmarshal(b, &md); err != nil { + return metadata.Metadata{}, err + } + return md, nil +} + +func (mgr *Manager) metadataUpdate(ctx context.Context, old metadata.Metadata) (metadata.Metadata, error) { + return metadata.Metadata{}, fmt.Errorf("TODO: update metadata") +} diff --git a/internal/merkle/TODO b/internal/merkle/TODO new file mode 100644 index 0000000..46cc0cb --- /dev/null +++ b/internal/merkle/TODO @@ -0,0 +1 @@ +Drop this package, fix the minor edit in upstream. diff --git a/internal/merkle/compact.go b/internal/merkle/compact.go new file mode 100644 index 0000000..6eeabd0 --- /dev/null +++ b/internal/merkle/compact.go @@ -0,0 +1,115 @@ +// BSD 2-Clause License +// +// Copyright (c) 2022, the ct authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// From: +// https://gitlab.torproject.org/rgdd/ct/-/tree/main/pkg/merkle +// +// The only difference is that leaf hashes rather than leaf data are passed as +// input to TreeHeadFromRangeProof, thus also changing the nodes() helper. +package merkle + +import ( + "crypto/sha256" + "fmt" +) + +// node represents a subtree at some level and a particular index +type node struct { + index uint64 + hash [sha256.Size]byte +} + +// nodes returns a list of consecutive leaf hashes +func nodes(index uint64, leafHashes [][sha256.Size]byte) (n []node) { + for i, lh := range leafHashes { + n = append(n, node{index + uint64(i), lh}) + } + return +} + +// compactRange outputs the minimal number of fixed subtree hashes given a +// non-empty list of consecutive leaves that start from a non-zero index. For a +// definition of this algorithm, see the end of ../../doc/tlog_algorithms.md. +func compactRange(nodes []node) [][sha256.Size]byte { + // Step 1 + var hashes [][sha256.Size]byte + + // Step 2 + for len(nodes) > 1 { + // Step 2a + if xor(nodes[1].index, 1) != nodes[0].index { + hashes = append(hashes, nodes[0].hash) + nodes = nodes[1:] + } + + // Step 2b; Step 2c; Step 2c(iii) + for i := 0; i < len(nodes); i++ { + // Step 2c(i) + if i+1 != len(nodes) { + nodes[i].hash = HashInteriorNode(nodes[i].hash, nodes[i+1].hash) + nodes = append(nodes[:i+1], nodes[i+2:]...) + } + + // Step 2c(ii) + nodes[i].index = rshift(nodes[i].index) + } + } + + // Step 3 + return append(hashes, nodes[0].hash) +} + +// TreeHeadFromRangeProof computes a tree head at size n=len(leafHashes)+index +// if given a list of leaf hashes at indices index,...,n-1 as well as an +// inclusion proof for the first leaf in the tree of size n. This allows a +// verifier to check inclusion of one or more log entries with a single +// inclusion proof. +func TreeHeadFromRangeProof(leafHashes [][sha256.Size]byte, index uint64, proof [][sha256.Size]byte) (root [sha256.Size]byte, err error) { + var cr [][sha256.Size]byte + confirmHash := func(h [sha256.Size]byte) error { + if h != cr[0] { + return fmt.Errorf("aborted due incorrect right-node subtree hash") + } + cr = cr[1:] + return nil + } + copyRoot := func(r [sha256.Size]byte) error { + root = r + return nil + } + + if len(leafHashes) == 0 { + return [sha256.Size]byte{}, fmt.Errorf("need at least one leaf to recompute tree head from proof") + } + if len(leafHashes) > 1 { + cr = compactRange(nodes(index+1, leafHashes[1:])) + } + return root, inclusion(leafHashes[0], index, index+uint64(len(leafHashes)), proof, copyRoot, confirmHash) +} + +func xor(a, b uint64) uint64 { + return a ^ b +} diff --git a/internal/merkle/merkle.go b/internal/merkle/merkle.go new file mode 100644 index 0000000..872364f --- /dev/null +++ b/internal/merkle/merkle.go @@ -0,0 +1,271 @@ +// BSD 2-Clause License +// +// Copyright (c) 2022, the ct authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// From: +// https://gitlab.torproject.org/rgdd/ct/-/tree/main/pkg/merkle +package merkle + +import ( + "crypto/sha256" + "fmt" +) + +// HashEmptyTree computes the hash of an empty tree. See RFC 6162, §2.1: +// +// MTH({}) = SHA-256() +func HashEmptyTree() [sha256.Size]byte { + return sha256.Sum256(nil) +} + +// HashLeafNode computes the hash of a leaf's data. See RFC 6162, §2.1: +// +// MTH({d(0)}) = SHA-256(0x00 || d(0)) +func HashLeafNode(data []byte) (hash [sha256.Size]byte) { + h := sha256.New() + h.Write([]byte{0x00}) + h.Write(data) + copy(hash[:], h.Sum(nil)) + return +} + +// HashInteriorNode computes the hash of an interior node. See RFC 6962, §2.1: +// +// MTH(D[n]) = SHA-256(0x01 || MTH(D[0:k]) || MTH(D[k:n]) +func HashInteriorNode(left, right [sha256.Size]byte) (hash [sha256.Size]byte) { + h := sha256.New() + h.Write([]byte{0x01}) + h.Write(left[:]) + h.Write(right[:]) + copy(hash[:], h.Sum(nil)) + return +} + +// inclusion implements the algorithm specified in RFC 9162, Section 2.1.3.2. +// In addition, the caller is allowed to confirm right-node subtree hashes. +func inclusion(leaf [sha256.Size]byte, index, size uint64, proof [][sha256.Size]byte, + confirmRoot func([sha256.Size]byte) error, confirmHash func([sha256.Size]byte) error) error { + // Step 1 + if index >= size { + return fmt.Errorf("leaf index must be in [%d, %d]", 0, size-1) + } + + // Step 2 + fn := index + sn := size - 1 + + // Step 3 + r := leaf + + // Step 4 + for i, p := range proof { + // Step 4a + if sn == 0 { + return fmt.Errorf("reached tree head with %d remaining proof hash(es)", len(proof[i:])) + } + + // Step 4b + if isLSB(fn) || fn == sn { + // Step 4b, i + r = HashInteriorNode(p, r) + + // Step 4b, ii + if !isLSB(fn) { + for { + fn = rshift(fn) + sn = rshift(sn) + + if isLSB(fn) || fn == 0 { + break + } + } + } + } else { + // Step 4b, i + r = HashInteriorNode(r, p) + + // Extension: allow the caller to confirm right-node subtree hashes + if err := confirmHash(p); err != nil { + return fmt.Errorf("subtree index %d: %v", fn, err) + } + } + + // Step 4c + fn = rshift(fn) + sn = rshift(sn) + } + + // Step 5 + if sn != 0 { + return fmt.Errorf("stopped at subtree with index %d due to missing proof hashes", fn) + } + return confirmRoot(r) +} + +// consistency implements the algorithm specified in RFC 9162, §2.1.4.2 +func consistency(oldSize, newSize uint64, oldRoot, newRoot [sha256.Size]byte, proof [][sha256.Size]byte) error { + // Step 1 + if len(proof) == 0 { + return fmt.Errorf("need at least one proof hash") + } + + // Step 2 + if isPOW2(oldSize) { + proof = append([][sha256.Size]byte{oldRoot}, proof...) + } + + // Step 3 + fn := oldSize - 1 + sn := newSize - 1 + + // Step 4 + for isLSB(fn) { + fn = rshift(fn) + sn = rshift(sn) + } + + // Step 5 + fr := proof[0] + sr := proof[0] + + // Step 6 + for i, c := range proof[1:] { + // Step 6a + if sn == 0 { + return fmt.Errorf("reached tree head with %d remaining proof hash(es)", len(proof[i+1:])) + } + + // Step 6b + if isLSB(fn) || fn == sn { + // Step 6b, i + fr = HashInteriorNode(c, fr) + // Step 6b, ii + sr = HashInteriorNode(c, sr) + // Step 6b, iii + if !isLSB(fn) { + for { + fn = rshift(fn) + sn = rshift(sn) + + if isLSB(fn) || fn == 0 { + break + } + } + } + } else { + // Step 6b, i + sr = HashInteriorNode(sr, c) + } + + // Step 6c + fn = rshift(fn) + sn = rshift(sn) + } + + // Step 7 + if sn != 0 { + return fmt.Errorf("stopped at subtree with index %d due to missing proof hashes", fn) + } + if fr != oldRoot { + return fmt.Errorf("recomputed old tree head %x is not equal to reference tree head %x", fr[:], oldRoot[:]) + } + if sr != newRoot { + return fmt.Errorf("recomputed new tree head %x is not equal to reference tree head %x", sr[:], newRoot[:]) + } + return nil +} + +// VerifyInclusion verifies that a leaf's data is commited at a given index in a +// reference tree +func VerifyInclusion(data []byte, index, size uint64, root [sha256.Size]byte, proof [][sha256.Size]byte) error { + if size == 0 { + return fmt.Errorf("tree size must be larger than zero") + } + + confirmHash := func(h [sha256.Size]byte) error { return nil } // No compact range extension + confirmRoot := func(r [sha256.Size]byte) error { + if r != root { + return fmt.Errorf("recomputed tree head %x is not equal to reference tree head %x", r[:], root[:]) + } + return nil + } + return inclusion(HashLeafNode(data), index, size, proof, confirmRoot, confirmHash) +} + +// VerifyConsistency verifies that an an old tree is consistent with a new tree +func VerifyConsistency(oldSize, newSize uint64, oldRoot, newRoot [sha256.Size]byte, proof [][sha256.Size]byte) error { + checkTree := func(size uint64, root [sha256.Size]byte) error { + if size == 0 { + if root != HashEmptyTree() { + return fmt.Errorf("non-empty tree head %x for size zero", root[:]) + } + if len(proof) != 0 { + return fmt.Errorf("non-empty proof with %d hashes for size zero", len(proof)) + } + } else if root == HashEmptyTree() { + return fmt.Errorf("empty tree head %x for tree size %d", root[:], size) + } + return nil + } + + if err := checkTree(oldSize, oldRoot); err != nil { + return fmt.Errorf("old: %v", err) + } + if err := checkTree(newSize, newRoot); err != nil { + return fmt.Errorf("new: %v", err) + } + if oldSize == 0 { + return nil + } + + if oldSize == newSize { + if oldRoot != newRoot { + return fmt.Errorf("different tree heads %x and %x with equal tree size %d", oldRoot, newRoot, oldSize) + } + if len(proof) != 0 { + return fmt.Errorf("non-empty proof with %d hashes for equal tree size %d", len(proof), oldSize) + } + return nil + } + if oldSize > newSize { + return fmt.Errorf("old tree size %d must be smaller than or equal to the new tree size %d", oldSize, newSize) + } + + return consistency(oldSize, newSize, oldRoot, newRoot, proof) +} + +// isLSB returns true if the least significant bit of num is set +func isLSB(num uint64) bool { + return (num & 1) != 0 +} + +// isPOW2 returns true if num is a power of two (1, 2, 4, 8, ...) +func isPOW2(num uint64) bool { + return (num & (num - 1)) == 0 +} + +func rshift(num uint64) uint64 { + return num >> 1 +} diff --git a/internal/options/options.go b/internal/options/options.go new file mode 100644 index 0000000..3e253c5 --- /dev/null +++ b/internal/options/options.go @@ -0,0 +1,97 @@ +package options + +import ( + "encoding/json" + "flag" + "fmt" + "os" + + "rgdd.se/silent-ct/internal/manager" + "rgdd.se/silent-ct/pkg/monitor" + "rgdd.se/silent-ct/pkg/server" +) + +const usage = `Usage: + + silent-ct [Options] + +Options: + + -h, --help: Output usage message and exit + -c, --config: Path to a configuration file (Default: %s) + -l, --listen: Listen address to receive submission on (Default: %s) + -s, --state: Path to a directory where state is stored (Default: %s) + +Example configuration file: + + { + "monitor": [ + { + "wildcard": "example.org", + "excludes": [ + "test" + ] + } + ], + "nodes": [ + { + "name": "node_a", + "secret": "aaaa", + "issues": [ + "example.org", + "www.example.org" + ] + } + ] + } + +` + +// Options are command-line options the user can specify +type Options struct { + ListenAddr string + ConfigFile string + StateDir string +} + +func New(cmd string, args []string) (opts Options, err error) { + fs := flag.NewFlagSet(cmd, flag.ContinueOnError) + fs.Usage = func() { + fmt.Fprintf(os.Stderr, usage, server.DefaultConfigFile, server.DefaultAddress, manager.DefaultStateDir) + } + stringOpt(fs, &opts.ConfigFile, "config", "c", server.DefaultConfigFile) + stringOpt(fs, &opts.ListenAddr, "listen", "l", server.DefaultAddress) + stringOpt(fs, &opts.StateDir, "state", "s", manager.DefaultStateDir) + if err = fs.Parse(args); err != nil { + return opts, err + } + + if opts.ConfigFile == "" { + return opts, fmt.Errorf("-c, --config: must not be an empty string") + } + if opts.StateDir == "" { + return opts, fmt.Errorf("-s, --state: must not be an empty string") + } + if opts.ListenAddr == "" { + return opts, fmt.Errorf("-l, --listen: must not be an empty string") + } + return opts, err +} + +func stringOpt(fs *flag.FlagSet, opt *string, short, long, value string) { + fs.StringVar(opt, short, value, "") + fs.StringVar(opt, long, value, "") +} + +type Config struct { + Monitor monitor.MatchWildcards `json:"monitor"` + Nodes server.Nodes `json:"nodes"` +} + +func (c *Config) FromFile(fileName string) error { + b, err := os.ReadFile(fileName) + if err != nil { + return err + } + return json.Unmarshal(b, c) +} diff --git a/main.go b/main.go index 4ffa7f3..b602222 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,108 @@ package main -import "fmt" +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "sync" + "syscall" + + "rgdd.se/silent-ct/internal/manager" + "rgdd.se/silent-ct/internal/options" + "rgdd.se/silent-ct/pkg/monitor" + "rgdd.se/silent-ct/pkg/server" +) func main() { - fmt.Println("TODO: silent-ct") + opts, err := options.New(os.Args[0], os.Args[1:]) + if err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + die("options: %v", err) + } + var c options.Config + if err := c.FromFile(opts.ConfigFile); err != nil { + die("configuration: %v", err) + } + + srv, err := server.New(server.Config{Address: opts.ListenAddr, Nodes: c.Nodes}) + if err != nil { + die("create new server: %v", err) + } + mon, err := monitor.New(monitor.Config{Callback: &c.Monitor}) + if err != nil { + die("create new monitor: %v", err) + } + mgr, err := manager.New(manager.Config{Nodes: c.Nodes}) + if err != nil { + die("create new manager: %v", err) + } + + configCh := make(chan []monitor.MessageLogConfig) + defer close(configCh) + + progressCh := make(chan monitor.MessageLogProgress) + defer close(progressCh) + + submitCh := make(chan server.MessageNodeSubmission) + defer close(submitCh) + + errorCh := make(chan error) + defer close(errorCh) + + ctx, cancel := context.WithCancel(context.Background()) + var wg sync.WaitGroup + defer wg.Wait() + + wg.Add(1) + go func() { + defer wg.Done() + defer cancel() + await(ctx) + }() + + wg.Add(1) + go func() { + defer wg.Done() + defer cancel() + if err := srv.Run(ctx, submitCh, errorCh); err != nil { + die("server: %v\n", err) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + defer cancel() + mon.Run(ctx, configCh, progressCh, errorCh) + }() + + wg.Add(1) + go func() { + defer wg.Done() + defer cancel() + if err := mgr.Run(ctx, submitCh, progressCh, configCh, errorCh); err != nil { + die("manager: %v\n", err) + } + }() +} + +func await(ctx context.Context) { + sigs := make(chan os.Signal, 1) + defer close(sigs) + + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + select { + case <-sigs: + case <-ctx.Done(): + } +} + +func die(format string, args ...interface{}) { + fmt.Printf("fatal: "+format, args) + os.Exit(1) } diff --git a/pkg/monitor/chunks.go b/pkg/monitor/chunks.go new file mode 100644 index 0000000..87871b9 --- /dev/null +++ b/pkg/monitor/chunks.go @@ -0,0 +1,88 @@ +package monitor + +// +// A min heap of chunks, oredered on each chunk's start index. +// +// Credit: inspiration to use a heap from Aaron Gable, see +// https://github.com/aarongable/ctaudit +// + +import ( + "container/heap" + "crypto/sha256" +) + +type chunk struct { + startIndex uint64 // Index of the first leaf + leafHashes [][sha256.Size]byte // List of consecutive leaf hashes + matches []LogEntry // Leaves that matches some criteria + errors []error // Errors that ocurred while parsing leaves +} + +type chunks []*chunk + +func newChunks() *chunks { + var h chunks + heap.Init((*internal)(&h)) + return &h +} + +func (h *chunks) push(c *chunk) { + heap.Push((*internal)(h), c) +} + +func (h *chunks) pop() *chunk { + x := heap.Pop((*internal)(h)) + return x.(*chunk) +} + +// gap returns true if there's a gap between the provided start index and the +// top most chunk. If the top most chunk is in sequence, it is merged with +// any following chunks that are also in sequence to form one larger chunk. +func (h *chunks) gap(start uint64) bool { + if len(*h) == 0 { + return true + } + + top := h.pop() + if start != top.startIndex { + h.push(top) + return true + } + + for len(*h) > 0 { + c := h.pop() + if c.startIndex != top.startIndex+uint64(len(top.leafHashes)) { + h.push(c) + break + } + + top.leafHashes = append(top.leafHashes, c.leafHashes...) + top.matches = append(top.matches, c.matches...) + top.errors = append(top.errors, c.errors...) + } + + h.push(top) + return false +} + +// internal implements the heap interface, see example: +// https://cs.opensource.google/go/go/+/refs/tags/go1.21.5:src/container/heap/example_intheap_test.go +type internal chunks + +func (h internal) Len() int { return len(h) } +func (h internal) Less(i, j int) bool { return h[i].startIndex < h[j].startIndex } +func (h internal) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *internal) Push(x any) { + *h = append(*h, x.(*chunk)) +} + +func (h *internal) Pop() any { + old := *h + n := len(old) + x := old[n-1] + old[n-1] = nil // avoid memory leak + *h = old[:n-1] + return x +} diff --git a/pkg/monitor/errors.go b/pkg/monitor/errors.go new file mode 100644 index 0000000..4d676af --- /dev/null +++ b/pkg/monitor/errors.go @@ -0,0 +1,41 @@ +package monitor + +import ( + "fmt" + + ct "github.com/google/certificate-transparency-go" +) + +// ErrorFetch occurs if there's a problem hitting the log's HTTP API. An STH is +// provided if available, since it might carry evidence of some log misbehavior. +type ErrorFetch struct { + URL string + Msg string + Err error + STH *ct.SignedTreeHead +} + +func (e ErrorFetch) Error() string { + return fmt.Sprintf("%s: %s: %v", e.URL, e.Msg, e.Err) +} + +// ErrorMerkleTree occurs if the log's Merkle tree can't be verified. An STH is +// provided if available (i.e., won't be available for internal tree building). +type ErrorMerkleTree struct { + URL string + Msg string + Err error + STH *ct.SignedTreeHead +} + +func (e ErrorMerkleTree) Error() string { + return fmt.Sprintf("%s: %s: %v", e.URL, e.Msg, e.Err) +} + +// TODO: MMD violations +// TODO: Growing read-only logs + +// noout implements the Logger interface to discard unwanted output +type noout struct{} + +func (n *noout) Printf(string, ...interface{}) {} diff --git a/pkg/monitor/matcher.go b/pkg/monitor/matcher.go new file mode 100644 index 0000000..fa3a894 --- /dev/null +++ b/pkg/monitor/matcher.go @@ -0,0 +1,90 @@ +package monitor + +import ( + "fmt" + "strings" + + ct "github.com/google/certificate-transparency-go" +) + +type Matcher interface { + Match(leafInput, extraData []byte) (bool, error) +} + +// MatchAll matches all certificates +type MatchAll struct{} + +func (m *MatchAll) Match(leafInput, extraData []byte) (bool, error) { + return true, nil +} + +// MatchWildcards matches a list of wildcards, see the MatchWildcard type +type MatchWildcards []MatchWildcard + +func (m *MatchWildcards) Match(leafInput, extraData []byte) (bool, error) { + sans, err := getSANs(ct.LeafEntry{LeafInput: leafInput, ExtraData: extraData}) + if err != nil { + return false, err + } + return m.match(sans), nil +} + +func (m *MatchWildcards) match(sans []string) bool { + for _, mw := range (*m)[:] { + if mw.match(sans) { + return true + } + } + return false +} + +// MatchWildcard exclude matches for `.*\.`, but will +// otherwise match on any `.*\.` as well as SANs equal to . +// +// For example, let be example.org and Exclude be [foo, bar]. Then +// example.org and www.example.org would match, whereas foo.example.org, +// sub.foo.example.org, and bar.example.org. would not match. +type MatchWildcard struct { + Wildcard string `json:"wildcard"` + Excludes []string `json:"excludes"` +} + +func (m *MatchWildcard) match(sans []string) bool { + for _, san := range sans { + if san == m.Wildcard { + return true + } + if strings.HasSuffix(san, "."+m.Wildcard) && !m.exclude(san) { + return true + } + } + return false +} + +func (m *MatchWildcard) exclude(san string) bool { + for _, exclude := range m.Excludes { + suffix := exclude + "." + m.Wildcard + if strings.HasSuffix(san, suffix) { + return true + } + } + return false +} + +func getSANs(entry ct.LeafEntry) ([]string, error) { + // Warning: here be dragons, parsing of DNS names in certificates... + e, err := ct.LogEntryFromLeaf(0, &entry) + if err != nil { + return nil, fmt.Errorf("parse leaf: %v", err) + } + if e.Precert == nil && e.X509Cert == nil { + return nil, fmt.Errorf("neither precertificate nor certificate in leaf") + } + if e.Precert != nil && e.X509Cert != nil { + return nil, fmt.Errorf("both certificate and precertificate in leaf") + } + if e.Precert != nil { + return e.Precert.TBSCertificate.DNSNames, nil + } + return e.X509Cert.DNSNames, nil +} diff --git a/pkg/monitor/messages.go b/pkg/monitor/messages.go new file mode 100644 index 0000000..717aae6 --- /dev/null +++ b/pkg/monitor/messages.go @@ -0,0 +1,40 @@ +package monitor + +import ( + ct "github.com/google/certificate-transparency-go" + "gitlab.torproject.org/rgdd/ct/pkg/metadata" +) + +// MessageLogConfig provides information about a log the monitor is downloading +type MessageLogConfig struct { + Metadata metadata.Log + State MonitorState +} + +// MessageLogProgress is the next log state and any encountered leaves that were +// considered matching since the previous log state. Parse errors are included. +type MessageLogProgress struct { + State MonitorState + Matches []LogEntry + Errors []error +} + +// MonitorState describes the monitor's state for a particular log. The signed tree +// head is the latest verified append-only state that was observed. The index +// is the next leaf which will be downloaded and processed by the monitor. +type MonitorState struct { + LogState + NextIndex uint64 +} + +// LogState describes the state of a log +type LogState struct { + ct.SignedTreeHead +} + +// LogEntry is a leaf in a log's Merkle tree +type LogEntry struct { + LeafIndex uint64 + LeafData []byte + ExtraData []byte +} diff --git a/pkg/monitor/monitor.go b/pkg/monitor/monitor.go new file mode 100644 index 0000000..5f7a629 --- /dev/null +++ b/pkg/monitor/monitor.go @@ -0,0 +1,286 @@ +// Package monitor provides a Certificate Transparency monitor that tails a list +// of logs which can be updated dynamically while running. All emitted progress +// messages have been verified by the monitor to be included in the log's +// append-only Merkle tree with regard to the initial start-up state. It is up +// to the user to process the monitor's progress, errors, and persist state. +// +// Implement the Matcher interface to customize which certificates should be +// included in a log's progress messages, or use any of the existing matchers +// provided by this package (see for example MatchAll and MatchWildcards). +package monitor + +import ( + "context" + "crypto/sha256" + "fmt" + "net/http" + "sync" + "time" + + ct "github.com/google/certificate-transparency-go" + "github.com/google/certificate-transparency-go/client" + "github.com/google/certificate-transparency-go/jsonclient" + "github.com/google/certificate-transparency-go/scanner" + "rgdd.se/silent-ct/internal/merkle" +) + +const ( + UserAgentPrefix = "rgdd.se/silent-ct" + DefaultContact = "unknown-user" + DefaultChunkSize = 256 // TODO: increase me + DefaultBatchSize = 128 // TODO: increase me + DefaultNumWorkers = 2 +) + +type Config struct { + Contact string // Something that help log operators get in touch + ChunkSize int // Min number of leaves to propagate a chunk without matches + BatchSize int // Max number of certificates to accept per worker + NumWorkers int // Number of parallel workers to use for each log + + // Callback determines which certificates are interesting to detect + Callback Matcher +} + +type Monitor struct { + Config +} + +func New(cfg Config) (Monitor, error) { + if cfg.Contact == "" { + cfg.Contact = "unknown-user" + } + if cfg.ChunkSize <= 0 { + cfg.ChunkSize = DefaultChunkSize + } + if cfg.BatchSize <= 0 { + cfg.BatchSize = DefaultBatchSize + } + if cfg.NumWorkers <= 0 { + cfg.NumWorkers = DefaultNumWorkers + } + if cfg.Callback == nil { + cfg.Callback = &MatchAll{} + } + return Monitor{Config: cfg}, nil +} + +func (mon *Monitor) Run(ctx context.Context, metadataCh chan []MessageLogConfig, eventCh chan MessageLogProgress, errorCh chan error) { + var wg sync.WaitGroup + var sctx context.Context + var cancel context.CancelFunc + + for { + select { + case <-ctx.Done(): + return + case metadata := <-metadataCh: + fmt.Printf("DEBUG: received new list with %d logs\n", len(metadata)) + if cancel != nil { + fmt.Printf("DEBUG: stopping all log tailers\n") + cancel() + wg.Wait() + } + + sctx, cancel = context.WithCancel(ctx) + for _, md := range metadata { + fmt.Printf("DEBUG: starting log tailer %s\n", md.Metadata.URL) + wg.Add(1) + go func(lcfg MessageLogConfig) { + defer wg.Done() + + opts := jsonclient.Options{Logger: &noout{}, UserAgent: UserAgentPrefix + ":" + mon.Contact} + cli, err := client.New(string(lcfg.Metadata.URL), &http.Client{}, opts) + if err != nil { + errorCh <- fmt.Errorf("unable to configure %s: %v", lcfg.Metadata.URL, err) + return + } + + chunkCh := make(chan *chunk) + defer close(chunkCh) + + t := tail{mon.Config, *cli, chunkCh, eventCh, errorCh} + if err := t.run(sctx, lcfg.State); err != nil { + errorCh <- fmt.Errorf("unable to continue tailing %s: %v", lcfg.Metadata.URL, err) + } + }(md) + } + } + } +} + +type tail struct { + mcfg Config + cli client.LogClient + + chunkCh chan *chunk + eventCh chan MessageLogProgress + errorCh chan error +} + +func (t *tail) run(ctx context.Context, state MonitorState) error { + var wg sync.WaitGroup + defer wg.Wait() + + mctx, cancel := context.WithCancel(ctx) + defer cancel() + + wg.Add(1) + go func() { + defer wg.Done() + defer cancel() + t.sequence(mctx, state) + }() + + fetcher := scanner.NewFetcher(&t.cli, &scanner.FetcherOptions{ + BatchSize: t.mcfg.BatchSize, + StartIndex: int64(state.NextIndex), + ParallelFetch: t.mcfg.NumWorkers, + Continuous: true, + }) + callback := func(eb scanner.EntryBatch) { + c := chunk{startIndex: uint64(eb.Start)} + for i := 0; i < len(eb.Entries); i++ { + c.leafHashes = append(c.leafHashes, merkle.HashLeafNode(eb.Entries[i].LeafInput)) + match, err := t.mcfg.Callback.Match(eb.Entries[i].LeafInput, eb.Entries[i].ExtraData) + if err != nil { + c.errors = append(c.errors, fmt.Errorf("while processing index %d for %s: %v", i, t.cli.BaseURI(), err)) + } else if match { + c.matches = append(c.matches, LogEntry{ + LeafIndex: uint64(i), + LeafData: eb.Entries[i].LeafInput, + ExtraData: eb.Entries[i].ExtraData, + }) + } + } + t.chunkCh <- &c + } + return fetcher.Run(mctx, callback) +} + +func (t *tail) sequence(ctx context.Context, state MonitorState) { + heap := newChunks() + for { + select { + case <-ctx.Done(): + return + case c := <-t.chunkCh: + heap.push(c) + if heap.gap(state.NextIndex) { + continue + } + c = heap.pop() + if len(c.matches) == 0 && len(c.leafHashes) < t.mcfg.ChunkSize { + heap.push(c) + continue // TODO: don't trigger if we havn't run nextState for too long + } + nextState, err := t.nextState(ctx, state, c) + if err != nil { + t.errorCh <- err + heap.push(c) + continue + } + + state = nextState + t.eventCh <- MessageLogProgress{State: state, Matches: c.matches, Errors: c.errors} + } + } +} + +func (t *tail) nextState(ctx context.Context, state MonitorState, c *chunk) (MonitorState, error) { + newState, err := t.nextConsistentState(ctx, state) + if err != nil { + return MonitorState{}, err + } + newState, err = t.nextIncludedState(ctx, state, c) + if err != nil { + return MonitorState{}, err + } + return newState, nil +} + +func (t *tail) nextConsistentState(ctx context.Context, state MonitorState) (MonitorState, error) { + sth, err := getSignedTreeHead(ctx, &t.cli) + if err != nil { + return MonitorState{}, ErrorFetch{URL: t.cli.BaseURI(), Msg: "get-sth", Err: err} + } + oldSize := state.TreeSize + oldRoot := state.SHA256RootHash + newSize := sth.TreeSize + newRoot := sth.SHA256RootHash + + proof, err := getConsistencyProof(ctx, &t.cli, oldSize, newSize) + if err != nil { + return MonitorState{}, ErrorFetch{URL: t.cli.BaseURI(), Msg: "get-consistency", STH: sth, Err: err} + } + if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(proof)); err != nil { + return MonitorState{}, ErrorMerkleTree{URL: t.cli.BaseURI(), Msg: "consistency", STH: sth, Err: err} + } + + fmt.Printf("DEBUG: consistently updated STH from size %d to %d\n", oldSize, newSize) + return MonitorState{LogState: LogState{*sth}, NextIndex: state.NextIndex}, nil +} + +func (t *tail) nextIncludedState(ctx context.Context, state MonitorState, c *chunk) (MonitorState, error) { + leafHash := c.leafHashes[0] + oldSize := state.NextIndex + uint64(len(c.leafHashes)) + iproof, err := getInclusionProof(ctx, &t.cli, leafHash, oldSize) + if err != nil { + err = fmt.Errorf("leaf hash %x and tree size %d: %v", leafHash[:], oldSize, err) + return MonitorState{}, ErrorFetch{URL: t.cli.BaseURI(), Msg: "get-inclusion", Err: err} + } + if got, want := uint64(iproof.LeafIndex), state.NextIndex; got != want { + err := fmt.Errorf("leaf hash %x and tree size %d: expected leaf index %d but got %d", leafHash[:], oldSize, got, want) + return MonitorState{}, ErrorMerkleTree{URL: t.cli.BaseURI(), Msg: "proof-index", Err: err} + } + oldRoot, err := merkle.TreeHeadFromRangeProof(c.leafHashes, state.NextIndex, unslice(iproof.AuditPath)) + if err != nil { + return MonitorState{}, ErrorMerkleTree{URL: t.cli.BaseURI(), Msg: "inclusion", Err: err} + } + + newSize := state.TreeSize + newRoot := state.SHA256RootHash + cproof, err := getConsistencyProof(ctx, &t.cli, oldSize, newSize) + if err != nil { + err = fmt.Errorf("from size %d to %d: %v", oldSize, newSize, err) + return MonitorState{}, ErrorFetch{URL: t.cli.BaseURI(), Msg: "get-consistency", Err: err} + } + if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(cproof)); err != nil { + err = fmt.Errorf("from size %d to %d: %v", oldSize, newSize, err) + return MonitorState{}, ErrorMerkleTree{URL: t.cli.BaseURI(), Msg: "consistency", Err: err} + } + + state.NextIndex += uint64(len(c.leafHashes)) + return state, nil +} + +func getInclusionProof(ctx context.Context, cli *client.LogClient, leafHash [sha256.Size]byte, size uint64) (*ct.GetProofByHashResponse, error) { + rctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return cli.GetProofByHash(rctx, leafHash[:], size) +} + +func getConsistencyProof(ctx context.Context, cli *client.LogClient, oldSize, newSize uint64) ([][]byte, error) { + if oldSize == 0 || oldSize >= newSize { + return [][]byte{}, nil + } + rctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return cli.GetSTHConsistency(rctx, oldSize, newSize) +} + +func getSignedTreeHead(ctx context.Context, cli *client.LogClient) (*ct.SignedTreeHead, error) { + rctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return cli.GetSTH(rctx) +} + +func unslice(hashes [][]byte) [][sha256.Size]byte { + var ret [][sha256.Size]byte + for _, hash := range hashes { + var h [sha256.Size]byte + copy(h[:], hash) + ret = append(ret, h) + } + return ret +} diff --git a/pkg/server/errors.go b/pkg/server/errors.go new file mode 100644 index 0000000..abb4e43 --- /dev/null +++ b/pkg/server/errors.go @@ -0,0 +1 @@ +package server diff --git a/pkg/server/messages.go b/pkg/server/messages.go new file mode 100644 index 0000000..50edded --- /dev/null +++ b/pkg/server/messages.go @@ -0,0 +1,4 @@ +package server + +type MessageNodeSubmission struct { +} diff --git a/pkg/server/server.go b/pkg/server/server.go new file mode 100644 index 0000000..2d10c4b --- /dev/null +++ b/pkg/server/server.go @@ -0,0 +1,106 @@ +package server + +import ( + "context" + "fmt" + "net" + "net/http" + "strings" + "time" +) + +const ( + EndpointAddChain = "/add-chain" + EndpointGetStatus = "/get-status" + + DefaultAddress = "localhost:2009" + DefaultConfigFile = "/home/rgdd/.config/silent-ct/config.json" // FIXME +) + +type Nodes []Node + +type Node struct { + Name string `json:"name"` + Secret string `json:"secret"` + Domains []string `json:"issues"` +} + +type Config struct { + Address string // hostname[:port] or unix:///path/to/file.sock + Nodes Nodes // Which nodes are trusted to issue what certificates +} + +type Server struct { + Config + http.Server + + unixSocket bool // true if listening with a unix socket + eventCh chan MessageNodeSubmission + errorCh chan error +} + +func New(cfg Config) (Server, error) { + mux := http.NewServeMux() + srv := Server{Config: cfg, Server: http.Server{Handler: mux}} + mux.HandleFunc(EndpointAddChain, func(w http.ResponseWriter, r *http.Request) { srv.addChain(w, r) }) + mux.HandleFunc(EndpointGetStatus, func(w http.ResponseWriter, r *http.Request) { srv.getStatus(w, r) }) + if len(srv.Address) == 0 { + srv.Config.Address = DefaultAddress + } + if strings.HasPrefix(srv.Config.Address, "unix://") { + srv.Config.Address = srv.Config.Address[7:] + srv.unixSocket = true + } + return srv, nil +} + +func (srv *Server) Run(ctx context.Context, submitCh chan MessageNodeSubmission, errorCh chan error) error { + srv.eventCh = submitCh + srv.errorCh = errorCh + network := "unix" + if !srv.unixSocket { + network = "tcp" + } + + listener, err := net.Listen(network, srv.Address) + if err != nil { + return fmt.Errorf("listen: %v", err) + } + defer listener.Close() + + exitErr := make(chan error, 1) + defer close(exitErr) + go func() { + exitErr <- srv.Serve(listener) + }() + + select { + case err := <-exitErr: + if err != nil && err != http.ErrServerClosed { + return fmt.Errorf("serve: %v", err) + } + case <-ctx.Done(): + tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(tctx); err != nil { + return fmt.Errorf("shutdown: %v", err) + } + } + return nil +} + +func (srv *Server) getStatus(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Only HTTP GET method is allowed", http.StatusMethodNotAllowed) + return + } + fmt.Fprintf(w, "OK\n") +} + +func (srv *Server) addChain(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Only HTTP POST method is allowed", http.StatusMethodNotAllowed) + return + } + fmt.Fprintf(w, "TODO: HTTP POST /add-chain\n") +} -- cgit v1.2.3