Only bootstrap a compact range once per log

As opposed to doing a new bootstrap with get-proof-by-hash every time
the next root is constructed.  Bootstrapping the compact range from a
get-proof-by-hash query works for the most part, but fails if the log
included a duplicate entry and gives us the index for that instead.  Log
operators with duplicate entries include Cloudflare and Digicert.

If bootstrap fails (unlucky), we try to bootstrap again once the log's
signed tree head moved forward (hoping the last entry has no duplicate).

The more reliable way to bootstrap a compact range would be to use the
get-entry-and-proof endpoint.  This does not work in practise because
some logs are not implementing this endpoint.  Digicert has such logs.

1 files changed, 17 insertions, 55 deletions

diff --git a/internal/monitor/tail.go b/internal/monitor/tail.go
index 0e16476..6be165b 100644
--- a/internal/monitor/tail.go
+++ b/internal/monitor/tail.go
@@ -2,15 +2,14 @@ package monitor
 
 import (
 	"context"
-	"crypto/sha256"
 	"fmt"
 	"sync"
-	"time"
 
-	ct "github.com/google/certificate-transparency-go"
 	"github.com/google/certificate-transparency-go/client"
 	"github.com/google/certificate-transparency-go/scanner"
 	"gitlab.torproject.org/rgdd/ct/pkg/merkle"
+	"rgdd.se/silent-ct/internal/ioutil"
+	"rgdd.se/silent-ct/internal/logutil"
 )
 
 type tail struct {
@@ -118,7 +117,7 @@ func (t *tail) nextState(ctx context.Context, state State, c *chunk) (State, err
 }
 
 func (t *tail) nextConsistentState(ctx context.Context, state State) (State, error) {
-	sth, err := getSignedTreeHead(ctx, t.checker)
+	sth, err := logutil.GetSignedTreeHead(ctx, t.checker)
 	if err != nil {
 		return State{}, fmt.Errorf("%s: get-sth: %v", t.checker.BaseURI(), err)
 	}
@@ -128,73 +127,36 @@ func (t *tail) nextConsistentState(ctx context.Context, state State) (State, err
 	newSize := sth.TreeSize
 	newRoot := sth.SHA256RootHash
 
-	proof, err := getConsistencyProof(ctx, t.checker, oldSize, newSize)
+	proof, err := logutil.GetConsistencyProof(ctx, t.checker, oldSize, newSize)
 	if err != nil {
 		return State{}, fmt.Errorf("%s: get-consistency: %v", t.checker.BaseURI(), err)
 	}
-	if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(proof)); err != nil {
+	if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, proof); err != nil {
 		return State{}, fmt.Errorf("%s: verify consistency: %v", t.checker.BaseURI(), err)
 	}
 
-	return State{SignedTreeHead: *sth, NextIndex: state.NextIndex}, nil
+	return State{SignedTreeHead: *sth, CompactRange: ioutil.CopyHashes(state.CompactRange), NextIndex: state.NextIndex}, nil
 }
 
 func (t *tail) nextIncludedState(ctx context.Context, state State, c *chunk) (State, error) {
-	leafHash := c.leafHashes[0]
-	oldSize := state.NextIndex + uint64(len(c.leafHashes))
-	iproof, err := getInclusionProof(ctx, t.checker, leafHash, oldSize) // FIXME: set leaf index in ctx to hack into tile API
-	if err != nil {
-		return State{}, fmt.Errorf("%s: get-inclusion: %v", t.checker.BaseURI(), err)
-	}
-	if got, want := uint64(iproof.LeafIndex), state.NextIndex; got != want {
-		return State{}, fmt.Errorf("%s: wrong index for get-inclusion proof query %x:%d", t.checker.BaseURI(), leafHash[:], oldSize)
-	}
-	oldRoot, err := merkle.TreeHeadFromRangeProof(c.leafHashes, state.NextIndex, unslice(iproof.AuditPath))
+	cr, err := logutil.AppendCompactRange(state.CompactRange, state.NextIndex, c.leafHashes)
 	if err != nil {
-		return State{}, fmt.Errorf("%s: range proof: %v", t.checker.BaseURI(), err)
+		panic(fmt.Sprintf("bug: %v", err))
 	}
-
-	newSize := state.TreeSize
+	oldRoot := logutil.RootHash(cr)
+	oldSize := state.NextIndex + uint64(len(c.leafHashes))
 	newRoot := state.SHA256RootHash
-	cproof, err := getConsistencyProof(ctx, t.checker, oldSize, newSize)
+	newSize := state.TreeSize
+
+	proof, err := logutil.GetConsistencyProof(ctx, t.checker, oldSize, newSize)
 	if err != nil {
-		return State{}, fmt.Errorf("%s: get-consistency: %v", t.checker.BaseURI(), err)
+		return State{}, fmt.Errorf("%s: tree: get-consistency: %v", t.checker.BaseURI(), err)
 	}
-	if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(cproof)); err != nil {
-		return State{}, fmt.Errorf("%s: verify consistency: %v", t.checker.BaseURI(), err)
+	if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, proof); err != nil {
+		return State{}, fmt.Errorf("%s: tree: verify consistency: %v", t.checker.BaseURI(), err)
 	}
 
 	state.NextIndex += uint64(len(c.leafHashes))
+	state.CompactRange = ioutil.UnsliceHashes(cr.Hashes())
 	return state, nil
 }
-
-func getInclusionProof(ctx context.Context, cli client.CheckLogClient, leafHash [sha256.Size]byte, size uint64) (*ct.GetProofByHashResponse, error) {
-	rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-	return cli.GetProofByHash(rctx, leafHash[:], size)
-}
-
-func getConsistencyProof(ctx context.Context, cli client.CheckLogClient, oldSize, newSize uint64) ([][]byte, error) {
-	if oldSize == 0 || oldSize >= newSize {
-		return [][]byte{}, nil
-	}
-	rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-	return cli.GetSTHConsistency(rctx, oldSize, newSize)
-}
-
-func getSignedTreeHead(ctx context.Context, cli client.CheckLogClient) (*ct.SignedTreeHead, error) {
-	rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-	return cli.GetSTH(rctx)
-}
-
-func unslice(hashes [][]byte) [][sha256.Size]byte {
-	var ret [][sha256.Size]byte
-	for _, hash := range hashes {
-		var h [sha256.Size]byte
-		copy(h[:], hash)
-		ret = append(ret, h)
-	}
-	return ret
-}