fix: Ensure rate-limits work for get-entries

See further details here:
https://github.com/google/certificate-transparency-go/issues/898

We're doing the same fix as silentct until upstream is fixed:
https://git.glasklar.is/rgdd/silentct/-/blob/main/internal/monitor/backoff.go

2 files changed, 57 insertions, 1 deletions

diff --git a/backoff.go b/backoff.go
new file mode 100644
index 0000000..4fba942
--- /dev/null
+++ b/backoff.go
@@ -0,0 +1,56 @@
+package main
+
+//
+// Source:
+// https://git.glasklar.is/rgdd/silentct/-/blob/main/internal/monitor/backoff.go
+//
+
+import (
+	"context"
+
+	ct "github.com/google/certificate-transparency-go"
+	"github.com/google/certificate-transparency-go/client"
+	"github.com/google/certificate-transparency-go/jsonclient"
+	"github.com/google/trillian/client/backoff"
+)
+
+// backoffClient wraps client.LogClient so that we always backoff on get-entries
+// 4XX and 5XX.  Backoff is on by default for get-sth already, and our ct-sans
+// usage is not overly aggressive on proof endpoints (max once per new chunk).
+//
+// For reference on why this temporary work around is needed, see:
+// https://github.com/google/certificate-transparency-go/issues/898
+type backoffClient struct {
+	cli *client.LogClient
+}
+
+func (bc *backoffClient) BaseURI() string {
+	return bc.cli.BaseURI()
+}
+
+func (bc *backoffClient) GetSTH(ctx context.Context) (*ct.SignedTreeHead, error) {
+	return bc.cli.GetSTH(ctx)
+}
+
+func (bc *backoffClient) GetSTHConsistency(ctx context.Context, first, second uint64) ([][]byte, error) {
+	return bc.cli.GetSTHConsistency(ctx, first, second)
+}
+
+func (bc *backoffClient) GetProofByHash(ctx context.Context, hash []byte, treeSize uint64) (*ct.GetProofByHashResponse, error) {
+	return bc.cli.GetProofByHash(ctx, hash, treeSize)
+}
+
+func (bc *backoffClient) GetRawEntries(ctx context.Context, start, end int64) (*ct.GetEntriesResponse, error) {
+	rsp, err := bc.cli.GetRawEntries(ctx, start, end)
+	if err != nil {
+		jcErr, ok := err.(jsonclient.RspError)
+		if !ok {
+			return rsp, err
+		}
+		if jcErr.StatusCode < 400 || jcErr.StatusCode >= 600 {
+			return rsp, err
+		}
+		return rsp, backoff.RetriableErrorf("get-entries: %v", err)
+	}
+	return rsp, err
+}
diff --git a/cmd_collect.go b/cmd_collect.go
index 742884a..79043a1 100644
--- a/cmd_collect.go
+++ b/cmd_collect.go
@@ -88,7 +88,7 @@ func collect(opts options) error {
 				cancel()
 				return
 			}
-			fetcher := scanner.NewFetcher(cli, &scanner.FetcherOptions{
+			fetcher := scanner.NewFetcher(&backoffClient{cli: cli}, &scanner.FetcherOptions{
 				BatchSize:     int(opts.BatchSize),
 				StartIndex:    th.TreeSize,
 				EndIndex:      int64(sth.TreeSize),