Only bootstrap a compact range once per log

As opposed to doing a new bootstrap with get-proof-by-hash every time
the next root is constructed.  Bootstrapping the compact range from a
get-proof-by-hash query works for the most part, but fails if the log
included a duplicate entry and gives us the index for that instead.  Log
operators with duplicate entries include Cloudflare and Digicert.

If bootstrap fails (unlucky), we try to bootstrap again once the log's
signed tree head moved forward (hoping the last entry has no duplicate).

The more reliable way to bootstrap a compact range would be to use the
get-entry-and-proof endpoint.  This does not work in practise because
some logs are not implementing this endpoint.  Digicert has such logs.

1 files changed, 108 insertions, 0 deletions

diff --git a/internal/logutil/logutil.go b/internal/logutil/logutil.go
new file mode 100644
index 0000000..27c3a73
--- /dev/null
+++ b/internal/logutil/logutil.go
@@ -0,0 +1,108 @@
+// Package logutil wraps functions related to a log's Merkle tree.  All log
+// queries use a context where the deadline is set to a hardcoded value.
+package logutil
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"time"
+
+	ct "github.com/google/certificate-transparency-go"
+	"github.com/google/certificate-transparency-go/client"
+	"github.com/transparency-dev/merkle/compact"
+	"github.com/transparency-dev/merkle/rfc6962"
+	"gitlab.torproject.org/rgdd/ct/pkg/merkle"
+	"rgdd.se/silent-ct/internal/ioutil"
+)
+
+const timeout = 10 * time.Second
+
+func GetSignedTreeHead(ctx context.Context, cli client.CheckLogClient) (*ct.SignedTreeHead, error) {
+	rctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	return cli.GetSTH(rctx)
+}
+
+func GetConsistencyProof(ctx context.Context, cli client.CheckLogClient, oldSize, newSize uint64) ([][sha256.Size]byte, error) {
+	if oldSize > newSize {
+		return nil, fmt.Errorf("old size %d is larger than new size %d", oldSize, newSize)
+	}
+	if oldSize == 0 || oldSize == newSize {
+		return [][sha256.Size]byte{}, nil
+	}
+
+	rctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	proof, err := cli.GetSTHConsistency(rctx, oldSize, newSize)
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.UnsliceHashes(proof), nil
+}
+
+func GetEntry(ctx context.Context, cli *client.LogClient, index uint64) (*ct.LeafEntry, error) {
+	rctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	rsp, err := cli.GetRawEntries(rctx, int64(index), int64(index))
+	if err != nil {
+		return nil, err
+	}
+	if got, want := len(rsp.Entries), 1; got != want {
+		return nil, fmt.Errorf("too many log entries: %d (want one)", got)
+	}
+	return &rsp.Entries[0], nil
+}
+
+// GetCompactRange constructs the compact range [0, index) by doing a
+// get-proof-by-hash query to obtain the necessary tree hashes
+func GetCompactRange(ctx context.Context, cli *client.LogClient, entry *ct.LeafEntry, index uint64) (*compact.Range, error) {
+	rctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	leafHash := merkle.HashLeafNode(entry.LeafInput)
+	proof, err := cli.GetProofByHash(rctx, leafHash[:], index+1)
+	if err != nil {
+		return nil, err
+	}
+	if got, want := uint64(proof.LeafIndex), index; got != want {
+		return nil, fmt.Errorf("invalid leaf index: %d (want %d)", got, want)
+	}
+	tree := ioutil.UnsliceHashes(reverse(proof.AuditPath))
+	return AppendCompactRange(tree, index, [][sha256.Size]byte{leafHash})
+}
+
+// RootHash panics if the compact range is malformed
+func RootHash(cr *compact.Range) (ret [sha256.Size]byte) {
+	rootHash, err := cr.GetRootHash(nil)
+	if err != nil {
+		panic(fmt.Sprintf("bug: compact: %v", err))
+	}
+	if got, want := len(rootHash), sha256.Size; got != want {
+		panic(fmt.Sprintf("bug: invalid root hash: size %d", got))
+	}
+	copy(ret[:], rootHash[:])
+	return
+}
+
+// AppendCompactRange appends a list of leaf hashes to the compact range [0, size)
+func AppendCompactRange(tree [][sha256.Size]byte, size uint64, pending [][sha256.Size]byte) (*compact.Range, error) {
+	rf := compact.RangeFactory{Hash: rfc6962.DefaultHasher.HashChildren}
+	cr, err := rf.NewRange(0, size, ioutil.SliceHashes(tree))
+	if err != nil {
+		return nil, fmt.Errorf("compact: %v", err)
+	}
+	for _, leafHash := range pending {
+		dst := leafHash
+		cr.Append(dst[:], nil)
+	}
+	return cr, nil
+}
+
+func reverse(hashes [][]byte) (ret [][]byte) {
+	for i := len(hashes) - 1; i >= 0; i-- {
+		ret = append(ret, hashes[i])
+	}
+	return
+}