Only bootstrap a compact range once per log

As opposed to doing a new bootstrap with get-proof-by-hash every time
the next root is constructed.  Bootstrapping the compact range from a
get-proof-by-hash query works for the most part, but fails if the log
included a duplicate entry and gives us the index for that instead.  Log
operators with duplicate entries include Cloudflare and Digicert.

If bootstrap fails (unlucky), we try to bootstrap again once the log's
signed tree head moved forward (hoping the last entry has no duplicate).

The more reliable way to bootstrap a compact range would be to use the
get-entry-and-proof endpoint.  This does not work in practise because
some logs are not implementing this endpoint.  Digicert has such logs.

1 files changed, 66 insertions, 19 deletions

diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index 5e28aca..afd0bf0 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -8,13 +8,18 @@ import (
 	"crypto/x509"
 	"fmt"
 	"net/http"
+	"os"
 	"path/filepath"
 	"time"
 
+	ct "github.com/google/certificate-transparency-go"
 	"github.com/google/certificate-transparency-go/client"
 	"github.com/google/certificate-transparency-go/jsonclient"
+	"github.com/transparency-dev/merkle/compact"
 	"gitlab.torproject.org/rgdd/ct/pkg/metadata"
 	"rgdd.se/silent-ct/internal/ioutil"
+	"rgdd.se/silent-ct/internal/logger"
+	"rgdd.se/silent-ct/internal/logutil"
 	"rgdd.se/silent-ct/internal/monitor"
 	"rgdd.se/silent-ct/pkg/storage/index"
 	"rgdd.se/silent-ct/pkg/storage/loglist"
@@ -25,10 +30,12 @@ type Config struct {
 	Directory string // Path to a directory where everything will be stored
 
 	// Optional
-	AlertDelay  time.Duration     // Time before alerting on certificates that are unaccounted for
-	StaticLogs  []metadata.Log    // Static logs to configure in loglist
-	RemoveLogs  []metadata.LogKey // Keys of logs to omit in loglist
-	HTTPTimeout time.Duration     // HTTP timeout used when bootstrapping logs
+	Logger         *logger.Logger    // Info prints only (no output by default)
+	BootstrapTries uint              // The number of times to try bootstrapping a log before giving up
+	BootstrapWait  time.Duration     // How long to wait on bootstrap failures before trying again
+	AlertDelay     time.Duration     // Time before alerting on certificates that are unaccounted for
+	StaticLogs     []metadata.Log    // Static logs to configure in loglist
+	RemoveLogs     []metadata.LogKey // Keys of logs to omit in loglist
 }
 
 func (cfg *Config) CertificateIndexFile() string           { return cfg.Directory + "/crt_index.json" }
@@ -55,8 +62,14 @@ func (cfg *Config) configure() error {
 	if cfg.Directory == "" {
 		return fmt.Errorf("directory is required")
 	}
-	if cfg.HTTPTimeout == 0 {
-		cfg.HTTPTimeout = 10 * time.Second
+	if !cfg.Logger.IsConfigured() {
+		cfg.Logger = logger.New(logger.Config{Level: logger.LevelNotice, File: os.Stderr})
+	}
+	if cfg.BootstrapTries == 0 {
+		cfg.BootstrapTries = 64
+	}
+	if cfg.BootstrapWait == 0 {
+		cfg.BootstrapWait = 16 * time.Second
 	}
 
 	path, err := filepath.Abs(cfg.Directory)
@@ -110,31 +123,23 @@ func New(cfg Config) (Storage, error) {
 }
 
 func (s *Storage) BootstrapLog(ctx context.Context, log metadata.Log, skipBacklog bool) (monitor.State, error) {
-	storedState, err := s.GetMonitorState(log)
-	if err == nil {
+	if storedState, err := s.GetMonitorState(log); err == nil {
 		return storedState, ErrorMonitorStateExists
 	}
 
-	key, err := x509.MarshalPKIXPublicKey(log.Key.Public)
-	if err != nil {
-		return monitor.State{}, err
-	}
-	cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{PublicKeyDER: key})
+	cr, sth, err := s.bootstrapWithRetries(ctx, log)
 	if err != nil {
 		return monitor.State{}, err
 	}
-
-	sctx, cancel := context.WithTimeout(ctx, s.Config.HTTPTimeout)
-	defer cancel()
-	sth, err := cli.GetSTH(sctx)
+	id, err := log.Key.ID()
 	if err != nil {
-		return monitor.State{}, err
+		return monitor.State{}, fmt.Errorf("%s: %v", log.URL, err)
 	}
-	id, _ := log.Key.ID()
 	sth.LogID = id
 
 	state := monitor.State{SignedTreeHead: *sth}
 	if skipBacklog {
+		state.CompactRange = ioutil.UnsliceHashes(cr.Hashes())
 		state.NextIndex = sth.TreeSize
 	}
 	return state, s.SetMonitorState(id, state)
@@ -153,3 +158,45 @@ func (s *Storage) GetMonitorState(log metadata.Log) (monitor.State, error) {
 	state := monitor.State{}
 	return state, ioutil.ReadJSON(s.MonitorStateFile(id), &state)
 }
+
+func (s *Storage) bootstrapWithRetries(ctx context.Context, log metadata.Log) (*compact.Range, *ct.SignedTreeHead, error) {
+	for tries := s.BootstrapTries; tries > 0; tries-- {
+		cr, sth, err := s.bootstrap(ctx, log)
+		if err == nil {
+			return cr, sth, nil
+		}
+		if tries != 1 {
+			s.Logger.Infof("bootstrap: %v (retry in %v)\n", err, s.BootstrapWait)
+			time.Sleep(s.BootstrapWait)
+		}
+	}
+	return nil, nil, fmt.Errorf("failed to bootstrap after %d attempts: %s", s.BootstrapTries, log.URL)
+}
+
+func (s *Storage) bootstrap(ctx context.Context, log metadata.Log) (*compact.Range, *ct.SignedTreeHead, error) {
+	key, err := x509.MarshalPKIXPublicKey(log.Key.Public)
+	if err != nil {
+		return nil, nil, err
+	}
+	cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{PublicKeyDER: key})
+	if err != nil {
+		return nil, nil, err
+	}
+	sth, err := logutil.GetSignedTreeHead(ctx, cli)
+	if err != nil {
+		return nil, nil, fmt.Errorf("%s: get-sth: %v", log.URL, err)
+	}
+	entry, err := logutil.GetEntry(ctx, cli, sth.TreeSize-1)
+	if err != nil {
+		return nil, nil, fmt.Errorf("%s: get-entry: %v", log.URL, err)
+	}
+	cr, err := logutil.GetCompactRange(ctx, cli, entry, sth.TreeSize-1)
+	if err != nil {
+		return nil, nil, fmt.Errorf("%s: %v", log.URL, err)
+	}
+	rootHash := logutil.RootHash(cr)
+	if got, want := rootHash, sth.SHA256RootHash; got != want {
+		return nil, nil, fmt.Errorf("invalid root hash %x (want %x)", rootHash[:], sth.SHA256RootHash[:])
+	}
+	return cr, sth, nil
+}