fix: Ensure rate-limits are on for get-entriesmain

Backoff on 4XX and 5XX.  See related issue:
https://github.com/google/certificate-transparency-go/issues/898

Test manually hints:

```
$ cat srv.py
from http.server import HTTPServer, BaseHTTPRequestHandler

class RequestHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        self.send_response(429)
        self.send_header("Content-Type", "text/plain")
        self.end_headers()
        self.wfile.write(b"429 something something...")

    def do_POST(self):
        self.do_GET()

    def do_PUT(self):
        self.do_GET()

    def do_DELETE(self):
        self.do_GET()

if __name__ == "__main__":
    server_address = ('localhost', 9090)
    httpd = HTTPServer(server_address, RequestHandler)
    print("Server running on http://localhost:9090")
    httpd.serve_forever()
```

And a transport for http.Client that redirects to localhost:

```
type statusRR struct {
	inner http.RoundTripper
}

func (s *statusRR) RoundTrip(req *http.Request) (*http.Response, error) {
	if strings.Contains(req.URL.Path, "ct/v1/get-entries") {
		req.URL.Scheme = "http"
		req.URL.Host = "localhost:9090"
	}
	rsp, err := s.inner.RoundTrip(req)
	return rsp, err
}
```

1 files changed, 56 insertions, 0 deletions

diff --git a/internal/monitor/backoff.go b/internal/monitor/backoff.go
new file mode 100644
index 0000000..63c5f55
--- /dev/null
+++ b/internal/monitor/backoff.go
@@ -0,0 +1,56 @@
+package monitor
+
+import (
+	"context"
+
+	ct "github.com/google/certificate-transparency-go"
+	"github.com/google/certificate-transparency-go/client"
+	"github.com/google/certificate-transparency-go/jsonclient"
+	"github.com/google/trillian/client/backoff"
+)
+
+// backoffClient wraps client.LogClient so that we always backoff on get-entries
+// 4XX and 5XX.  Backoff is on by default for get-sth already, and our silentct
+// usage is guaranteed to not do any hammering on any of the proof endpoints.
+//
+// For reference on this issue, see:
+// https://github.com/google/certificate-transparency-go/issues/898
+type backoffClient struct {
+	cli *client.LogClient
+}
+
+func (bc *backoffClient) BaseURI() string {
+	return bc.cli.BaseURI()
+}
+
+func (bc *backoffClient) GetSTH(ctx context.Context) (*ct.SignedTreeHead, error) {
+	return bc.cli.GetSTH(ctx)
+}
+
+func (bc *backoffClient) GetSTHConsistency(ctx context.Context, first, second uint64) ([][]byte, error) {
+	return bc.cli.GetSTHConsistency(ctx, first, second)
+}
+
+func (bc *backoffClient) GetProofByHash(ctx context.Context, hash []byte, treeSize uint64) (*ct.GetProofByHashResponse, error) {
+	return bc.cli.GetProofByHash(ctx, hash, treeSize)
+}
+
+func (bc *backoffClient) GetRawEntries(ctx context.Context, start, end int64) (*ct.GetEntriesResponse, error) {
+	rsp, err := bc.cli.GetRawEntries(ctx, start, end)
+	if err != nil {
+		jcErr, ok := err.(jsonclient.RspError)
+		if !ok {
+			return rsp, err
+		}
+		if jcErr.StatusCode < 400 || jcErr.StatusCode >= 600 {
+			return rsp, err
+		}
+		// This ensures we never start hammering when the status code is 4XX or
+		// 5XX.  Probably not the right thing to do in all cases, but since the
+		// download library we're using starts hammering if the log suddenly
+		// serves something unexpected this seems like a good safety precaution.
+		// Users of the silentct monitor eventually notice they get no entries.
+		return rsp, backoff.RetriableErrorf("get-entries: %v", err)
+	}
+	return rsp, err
+}