aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--internal/ioutil/ioutil.go32
-rw-r--r--internal/logutil/logutil.go108
-rw-r--r--internal/manager/manager.go1
-rw-r--r--internal/monitor/monitor.go7
-rw-r--r--internal/monitor/tail.go72
-rw-r--r--pkg/storage/storage.go85
6 files changed, 229 insertions, 76 deletions
diff --git a/internal/ioutil/ioutil.go b/internal/ioutil/ioutil.go
index 7fe6cfc..8e98c65 100644
--- a/internal/ioutil/ioutil.go
+++ b/internal/ioutil/ioutil.go
@@ -1,6 +1,7 @@
package ioutil
import (
+ "crypto/sha256"
"encoding/json"
"fmt"
"os"
@@ -54,3 +55,34 @@ func DirectoriesExist(paths []string) error {
}
return nil
}
+
+func CopyHashes(hashes [][sha256.Size]byte) (ret [][sha256.Size]byte) {
+ for _, hash := range hashes {
+ var dst [sha256.Size]byte
+ copy(dst[:], hash[:])
+ ret = append(ret, dst)
+ }
+ return
+}
+
+func SliceHashes(hashes [][sha256.Size]byte) (ret [][]byte) {
+ for _, hash := range hashes {
+ dst := hash
+ ret = append(ret, dst[:])
+ }
+ return
+}
+
+// UnsliceHashes panics unless all hashes are 32 bytes
+func UnsliceHashes(hashes [][]byte) (ret [][sha256.Size]byte) {
+ for _, hash := range hashes {
+ if got, want := len(hash), sha256.Size; got != want {
+ panic(fmt.Sprintf("bug: invalid hash: size %d", got))
+ }
+
+ var dst [sha256.Size]byte
+ copy(dst[:], hash)
+ ret = append(ret, dst)
+ }
+ return
+}
diff --git a/internal/logutil/logutil.go b/internal/logutil/logutil.go
new file mode 100644
index 0000000..27c3a73
--- /dev/null
+++ b/internal/logutil/logutil.go
@@ -0,0 +1,108 @@
+// Package logutil wraps functions related to a log's Merkle tree. All log
+// queries use a context where the deadline is set to a hardcoded value.
+package logutil
+
+import (
+ "context"
+ "crypto/sha256"
+ "fmt"
+ "time"
+
+ ct "github.com/google/certificate-transparency-go"
+ "github.com/google/certificate-transparency-go/client"
+ "github.com/transparency-dev/merkle/compact"
+ "github.com/transparency-dev/merkle/rfc6962"
+ "gitlab.torproject.org/rgdd/ct/pkg/merkle"
+ "rgdd.se/silent-ct/internal/ioutil"
+)
+
+const timeout = 10 * time.Second
+
+func GetSignedTreeHead(ctx context.Context, cli client.CheckLogClient) (*ct.SignedTreeHead, error) {
+ rctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ return cli.GetSTH(rctx)
+}
+
+func GetConsistencyProof(ctx context.Context, cli client.CheckLogClient, oldSize, newSize uint64) ([][sha256.Size]byte, error) {
+ if oldSize > newSize {
+ return nil, fmt.Errorf("old size %d is larger than new size %d", oldSize, newSize)
+ }
+ if oldSize == 0 || oldSize == newSize {
+ return [][sha256.Size]byte{}, nil
+ }
+
+ rctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ proof, err := cli.GetSTHConsistency(rctx, oldSize, newSize)
+ if err != nil {
+ return nil, err
+ }
+ return ioutil.UnsliceHashes(proof), nil
+}
+
+func GetEntry(ctx context.Context, cli *client.LogClient, index uint64) (*ct.LeafEntry, error) {
+ rctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+
+ rsp, err := cli.GetRawEntries(rctx, int64(index), int64(index))
+ if err != nil {
+ return nil, err
+ }
+ if got, want := len(rsp.Entries), 1; got != want {
+ return nil, fmt.Errorf("too many log entries: %d (want one)", got)
+ }
+ return &rsp.Entries[0], nil
+}
+
+// GetCompactRange constructs the compact range [0, index) by doing a
+// get-proof-by-hash query to obtain the necessary tree hashes
+func GetCompactRange(ctx context.Context, cli *client.LogClient, entry *ct.LeafEntry, index uint64) (*compact.Range, error) {
+ rctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+
+ leafHash := merkle.HashLeafNode(entry.LeafInput)
+ proof, err := cli.GetProofByHash(rctx, leafHash[:], index+1)
+ if err != nil {
+ return nil, err
+ }
+ if got, want := uint64(proof.LeafIndex), index; got != want {
+ return nil, fmt.Errorf("invalid leaf index: %d (want %d)", got, want)
+ }
+ tree := ioutil.UnsliceHashes(reverse(proof.AuditPath))
+ return AppendCompactRange(tree, index, [][sha256.Size]byte{leafHash})
+}
+
+// RootHash panics if the compact range is malformed
+func RootHash(cr *compact.Range) (ret [sha256.Size]byte) {
+ rootHash, err := cr.GetRootHash(nil)
+ if err != nil {
+ panic(fmt.Sprintf("bug: compact: %v", err))
+ }
+ if got, want := len(rootHash), sha256.Size; got != want {
+ panic(fmt.Sprintf("bug: invalid root hash: size %d", got))
+ }
+ copy(ret[:], rootHash[:])
+ return
+}
+
+// AppendCompactRange appends a list of leaf hashes to the compact range [0, size)
+func AppendCompactRange(tree [][sha256.Size]byte, size uint64, pending [][sha256.Size]byte) (*compact.Range, error) {
+ rf := compact.RangeFactory{Hash: rfc6962.DefaultHasher.HashChildren}
+ cr, err := rf.NewRange(0, size, ioutil.SliceHashes(tree))
+ if err != nil {
+ return nil, fmt.Errorf("compact: %v", err)
+ }
+ for _, leafHash := range pending {
+ dst := leafHash
+ cr.Append(dst[:], nil)
+ }
+ return cr, nil
+}
+
+func reverse(hashes [][]byte) (ret [][]byte) {
+ for i := len(hashes) - 1; i >= 0; i-- {
+ ret = append(ret, hashes[i])
+ }
+ return
+}
diff --git a/internal/manager/manager.go b/internal/manager/manager.go
index bf1ad92..ce31b1b 100644
--- a/internal/manager/manager.go
+++ b/internal/manager/manager.go
@@ -51,6 +51,7 @@ func New(cfg Config, fch chan []feedback.Event, mch chan monitor.Event, cch chan
s, err := storage.New(storage.Config{
Bootstrap: cfg.Bootstrap,
Directory: cfg.Directory,
+ Logger: cfg.Logger,
AlertDelay: cfg.AlertDelay,
StaticLogs: cfg.Policy.StaticLogs,
RemoveLogs: cfg.Policy.RemoveLogs,
diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go
index 6de7193..2fe4d88 100644
--- a/internal/monitor/monitor.go
+++ b/internal/monitor/monitor.go
@@ -11,6 +11,7 @@ package monitor
import (
"context"
+ "crypto/sha256"
"crypto/x509"
"encoding/base64"
"fmt"
@@ -32,10 +33,12 @@ type MonitoredLog struct {
}
// State is the latest append-only state the monitor observed from its local
-// vantage point. The next entry to download is specified by NextIndex.
+// vantage point. The compact range covers [0, NextIndex). The next entry to
+// download from the log is at index NextIndex.
type State struct {
ct.SignedTreeHead `json:"latest_sth"`
- NextIndex uint64 `json:"next_index"`
+ CompactRange [][sha256.Size]byte `json:"compact_range"`
+ NextIndex uint64 `json:"next_index"`
}
// Event carries the latest consistent monitor state, found matches, as well as
diff --git a/internal/monitor/tail.go b/internal/monitor/tail.go
index 0e16476..6be165b 100644
--- a/internal/monitor/tail.go
+++ b/internal/monitor/tail.go
@@ -2,15 +2,14 @@ package monitor
import (
"context"
- "crypto/sha256"
"fmt"
"sync"
- "time"
- ct "github.com/google/certificate-transparency-go"
"github.com/google/certificate-transparency-go/client"
"github.com/google/certificate-transparency-go/scanner"
"gitlab.torproject.org/rgdd/ct/pkg/merkle"
+ "rgdd.se/silent-ct/internal/ioutil"
+ "rgdd.se/silent-ct/internal/logutil"
)
type tail struct {
@@ -118,7 +117,7 @@ func (t *tail) nextState(ctx context.Context, state State, c *chunk) (State, err
}
func (t *tail) nextConsistentState(ctx context.Context, state State) (State, error) {
- sth, err := getSignedTreeHead(ctx, t.checker)
+ sth, err := logutil.GetSignedTreeHead(ctx, t.checker)
if err != nil {
return State{}, fmt.Errorf("%s: get-sth: %v", t.checker.BaseURI(), err)
}
@@ -128,73 +127,36 @@ func (t *tail) nextConsistentState(ctx context.Context, state State) (State, err
newSize := sth.TreeSize
newRoot := sth.SHA256RootHash
- proof, err := getConsistencyProof(ctx, t.checker, oldSize, newSize)
+ proof, err := logutil.GetConsistencyProof(ctx, t.checker, oldSize, newSize)
if err != nil {
return State{}, fmt.Errorf("%s: get-consistency: %v", t.checker.BaseURI(), err)
}
- if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(proof)); err != nil {
+ if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, proof); err != nil {
return State{}, fmt.Errorf("%s: verify consistency: %v", t.checker.BaseURI(), err)
}
- return State{SignedTreeHead: *sth, NextIndex: state.NextIndex}, nil
+ return State{SignedTreeHead: *sth, CompactRange: ioutil.CopyHashes(state.CompactRange), NextIndex: state.NextIndex}, nil
}
func (t *tail) nextIncludedState(ctx context.Context, state State, c *chunk) (State, error) {
- leafHash := c.leafHashes[0]
- oldSize := state.NextIndex + uint64(len(c.leafHashes))
- iproof, err := getInclusionProof(ctx, t.checker, leafHash, oldSize) // FIXME: set leaf index in ctx to hack into tile API
- if err != nil {
- return State{}, fmt.Errorf("%s: get-inclusion: %v", t.checker.BaseURI(), err)
- }
- if got, want := uint64(iproof.LeafIndex), state.NextIndex; got != want {
- return State{}, fmt.Errorf("%s: wrong index for get-inclusion proof query %x:%d", t.checker.BaseURI(), leafHash[:], oldSize)
- }
- oldRoot, err := merkle.TreeHeadFromRangeProof(c.leafHashes, state.NextIndex, unslice(iproof.AuditPath))
+ cr, err := logutil.AppendCompactRange(state.CompactRange, state.NextIndex, c.leafHashes)
if err != nil {
- return State{}, fmt.Errorf("%s: range proof: %v", t.checker.BaseURI(), err)
+ panic(fmt.Sprintf("bug: %v", err))
}
-
- newSize := state.TreeSize
+ oldRoot := logutil.RootHash(cr)
+ oldSize := state.NextIndex + uint64(len(c.leafHashes))
newRoot := state.SHA256RootHash
- cproof, err := getConsistencyProof(ctx, t.checker, oldSize, newSize)
+ newSize := state.TreeSize
+
+ proof, err := logutil.GetConsistencyProof(ctx, t.checker, oldSize, newSize)
if err != nil {
- return State{}, fmt.Errorf("%s: get-consistency: %v", t.checker.BaseURI(), err)
+ return State{}, fmt.Errorf("%s: tree: get-consistency: %v", t.checker.BaseURI(), err)
}
- if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(cproof)); err != nil {
- return State{}, fmt.Errorf("%s: verify consistency: %v", t.checker.BaseURI(), err)
+ if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, proof); err != nil {
+ return State{}, fmt.Errorf("%s: tree: verify consistency: %v", t.checker.BaseURI(), err)
}
state.NextIndex += uint64(len(c.leafHashes))
+ state.CompactRange = ioutil.UnsliceHashes(cr.Hashes())
return state, nil
}
-
-func getInclusionProof(ctx context.Context, cli client.CheckLogClient, leafHash [sha256.Size]byte, size uint64) (*ct.GetProofByHashResponse, error) {
- rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
- defer cancel()
- return cli.GetProofByHash(rctx, leafHash[:], size)
-}
-
-func getConsistencyProof(ctx context.Context, cli client.CheckLogClient, oldSize, newSize uint64) ([][]byte, error) {
- if oldSize == 0 || oldSize >= newSize {
- return [][]byte{}, nil
- }
- rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
- defer cancel()
- return cli.GetSTHConsistency(rctx, oldSize, newSize)
-}
-
-func getSignedTreeHead(ctx context.Context, cli client.CheckLogClient) (*ct.SignedTreeHead, error) {
- rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
- defer cancel()
- return cli.GetSTH(rctx)
-}
-
-func unslice(hashes [][]byte) [][sha256.Size]byte {
- var ret [][sha256.Size]byte
- for _, hash := range hashes {
- var h [sha256.Size]byte
- copy(h[:], hash)
- ret = append(ret, h)
- }
- return ret
-}
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index 5e28aca..afd0bf0 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -8,13 +8,18 @@ import (
"crypto/x509"
"fmt"
"net/http"
+ "os"
"path/filepath"
"time"
+ ct "github.com/google/certificate-transparency-go"
"github.com/google/certificate-transparency-go/client"
"github.com/google/certificate-transparency-go/jsonclient"
+ "github.com/transparency-dev/merkle/compact"
"gitlab.torproject.org/rgdd/ct/pkg/metadata"
"rgdd.se/silent-ct/internal/ioutil"
+ "rgdd.se/silent-ct/internal/logger"
+ "rgdd.se/silent-ct/internal/logutil"
"rgdd.se/silent-ct/internal/monitor"
"rgdd.se/silent-ct/pkg/storage/index"
"rgdd.se/silent-ct/pkg/storage/loglist"
@@ -25,10 +30,12 @@ type Config struct {
Directory string // Path to a directory where everything will be stored
// Optional
- AlertDelay time.Duration // Time before alerting on certificates that are unaccounted for
- StaticLogs []metadata.Log // Static logs to configure in loglist
- RemoveLogs []metadata.LogKey // Keys of logs to omit in loglist
- HTTPTimeout time.Duration // HTTP timeout used when bootstrapping logs
+ Logger *logger.Logger // Info prints only (no output by default)
+ BootstrapTries uint // The number of times to try bootstrapping a log before giving up
+ BootstrapWait time.Duration // How long to wait on bootstrap failures before trying again
+ AlertDelay time.Duration // Time before alerting on certificates that are unaccounted for
+ StaticLogs []metadata.Log // Static logs to configure in loglist
+ RemoveLogs []metadata.LogKey // Keys of logs to omit in loglist
}
func (cfg *Config) CertificateIndexFile() string { return cfg.Directory + "/crt_index.json" }
@@ -55,8 +62,14 @@ func (cfg *Config) configure() error {
if cfg.Directory == "" {
return fmt.Errorf("directory is required")
}
- if cfg.HTTPTimeout == 0 {
- cfg.HTTPTimeout = 10 * time.Second
+ if !cfg.Logger.IsConfigured() {
+ cfg.Logger = logger.New(logger.Config{Level: logger.LevelNotice, File: os.Stderr})
+ }
+ if cfg.BootstrapTries == 0 {
+ cfg.BootstrapTries = 64
+ }
+ if cfg.BootstrapWait == 0 {
+ cfg.BootstrapWait = 16 * time.Second
}
path, err := filepath.Abs(cfg.Directory)
@@ -110,31 +123,23 @@ func New(cfg Config) (Storage, error) {
}
func (s *Storage) BootstrapLog(ctx context.Context, log metadata.Log, skipBacklog bool) (monitor.State, error) {
- storedState, err := s.GetMonitorState(log)
- if err == nil {
+ if storedState, err := s.GetMonitorState(log); err == nil {
return storedState, ErrorMonitorStateExists
}
- key, err := x509.MarshalPKIXPublicKey(log.Key.Public)
- if err != nil {
- return monitor.State{}, err
- }
- cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{PublicKeyDER: key})
+ cr, sth, err := s.bootstrapWithRetries(ctx, log)
if err != nil {
return monitor.State{}, err
}
-
- sctx, cancel := context.WithTimeout(ctx, s.Config.HTTPTimeout)
- defer cancel()
- sth, err := cli.GetSTH(sctx)
+ id, err := log.Key.ID()
if err != nil {
- return monitor.State{}, err
+ return monitor.State{}, fmt.Errorf("%s: %v", log.URL, err)
}
- id, _ := log.Key.ID()
sth.LogID = id
state := monitor.State{SignedTreeHead: *sth}
if skipBacklog {
+ state.CompactRange = ioutil.UnsliceHashes(cr.Hashes())
state.NextIndex = sth.TreeSize
}
return state, s.SetMonitorState(id, state)
@@ -153,3 +158,45 @@ func (s *Storage) GetMonitorState(log metadata.Log) (monitor.State, error) {
state := monitor.State{}
return state, ioutil.ReadJSON(s.MonitorStateFile(id), &state)
}
+
+func (s *Storage) bootstrapWithRetries(ctx context.Context, log metadata.Log) (*compact.Range, *ct.SignedTreeHead, error) {
+ for tries := s.BootstrapTries; tries > 0; tries-- {
+ cr, sth, err := s.bootstrap(ctx, log)
+ if err == nil {
+ return cr, sth, nil
+ }
+ if tries != 1 {
+ s.Logger.Infof("bootstrap: %v (retry in %v)\n", err, s.BootstrapWait)
+ time.Sleep(s.BootstrapWait)
+ }
+ }
+ return nil, nil, fmt.Errorf("failed to bootstrap after %d attempts: %s", s.BootstrapTries, log.URL)
+}
+
+func (s *Storage) bootstrap(ctx context.Context, log metadata.Log) (*compact.Range, *ct.SignedTreeHead, error) {
+ key, err := x509.MarshalPKIXPublicKey(log.Key.Public)
+ if err != nil {
+ return nil, nil, err
+ }
+ cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{PublicKeyDER: key})
+ if err != nil {
+ return nil, nil, err
+ }
+ sth, err := logutil.GetSignedTreeHead(ctx, cli)
+ if err != nil {
+ return nil, nil, fmt.Errorf("%s: get-sth: %v", log.URL, err)
+ }
+ entry, err := logutil.GetEntry(ctx, cli, sth.TreeSize-1)
+ if err != nil {
+ return nil, nil, fmt.Errorf("%s: get-entry: %v", log.URL, err)
+ }
+ cr, err := logutil.GetCompactRange(ctx, cli, entry, sth.TreeSize-1)
+ if err != nil {
+ return nil, nil, fmt.Errorf("%s: %v", log.URL, err)
+ }
+ rootHash := logutil.RootHash(cr)
+ if got, want := rootHash, sth.SHA256RootHash; got != want {
+ return nil, nil, fmt.Errorf("invalid root hash %x (want %x)", rootHash[:], sth.SHA256RootHash[:])
+ }
+ return cr, sth, nil
+}
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-18 10:05:22 +0000