Add drafty prototype

15 files changed, 947 insertions, 632 deletions

diff --git a/internal/feedback/feedback.go b/internal/feedback/feedback.go
new file mode 100644
index 0000000..77431e0
--- /dev/null
+++ b/internal/feedback/feedback.go
@@ -0,0 +1,127 @@
+package feedback
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	"rgdd.se/silent-ct/internal/logger"
+	"rgdd.se/silent-ct/pkg/crtutil"
+	"rgdd.se/silent-ct/pkg/policy"
+	"rgdd.se/silent-ct/pkg/submission"
+)
+
+type Event struct {
+	NodeName string // Name of the node that generated a submission
+	PEMChain []byte // A certificate chain found in the submission
+}
+
+type Config struct {
+	Policy policy.Policy
+
+	// Optional
+	Logger       logger.Logger // Debug and info prints only (no output by default)
+	PullInterval time.Duration // How often nodes are pulled via HTTP GET
+	HTTPTimeout  time.Duration // Timeout to use when pulling nodes
+}
+
+type Feedback struct {
+	cfg     Config
+	nodes   []policy.Node
+	eventCh chan []Event
+}
+
+func New(cfg Config, eventCh chan []Event) (Feedback, error) {
+	if !cfg.Logger.IsConfigured() {
+		cfg.Logger = logger.New(logger.Config{Level: logger.LevelNotice, File: os.Stdout})
+	}
+	if cfg.PullInterval == 0 {
+		cfg.PullInterval = 1 * time.Hour
+	}
+	if cfg.HTTPTimeout == 0 {
+		cfg.HTTPTimeout = 10 * time.Second
+	}
+
+	for i, node := range cfg.Policy.Nodes {
+		if err := node.Validate(); err != nil {
+			return Feedback{}, fmt.Errorf("node %d: %v", i, err)
+		}
+	}
+	return Feedback{cfg: cfg, nodes: cfg.Policy.Nodes, eventCh: eventCh}, nil
+}
+
+// RunForever collects legitimately issued certificates from nodes
+func (fb *Feedback) RunForever(ctx context.Context) {
+	ticker := time.NewTicker(fb.cfg.PullInterval)
+	defer ticker.Stop()
+
+	fb.pullOnce(ctx)
+	select {
+	case <-ticker.C:
+		fb.pullOnce(ctx)
+	case <-ctx.Done():
+		return
+	}
+}
+
+func (fb *Feedback) pullOnce(ctx context.Context) {
+	fb.cfg.Logger.Debugf("pull %d nodes\n", len(fb.nodes))
+	for _, node := range fb.nodes {
+		data, err := fb.pull(ctx, node)
+		if err != nil {
+			fb.cfg.Logger.Debugf("failed to pull node %s: %v", node.Name, err)
+			continue
+		}
+
+		var events []Event
+		for _, pemChain := range data {
+			chain, err := crtutil.CertificateChainFromPEM(pemChain)
+			if err != nil {
+				fb.cfg.Logger.Infof("failed to parse certificate from node %s: %v", node.Name, err)
+				continue
+			}
+			if err := node.Authorize(chain[0].DNSNames); err != nil {
+				fb.cfg.Logger.Infof("%s\n", err.Error())
+				continue
+			}
+
+			events = append(events, Event{NodeName: node.Name, PEMChain: pemChain})
+		}
+
+		fb.eventCh <- events
+	}
+}
+
+func (fb *Feedback) pull(ctx context.Context, node policy.Node) ([][]byte, error) {
+	req, err := http.NewRequest(http.MethodGet, node.URL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("new request: %v", err)
+	}
+	req.WithContext(ctx)
+
+	cli := http.Client{Timeout: fb.cfg.HTTPTimeout}
+	rsp, err := cli.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("HTTP %s: %v", req.Method, err)
+	}
+	defer rsp.Body.Close()
+
+	b, err := io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read response: %v", err)
+	}
+	if rsp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%s: %s", rsp.Status, string(b))
+	}
+
+	s := submission.Submission(b)
+	data, err := s.Open(node)
+	if err != nil {
+		return nil, fmt.Errorf("open: %v", err)
+	}
+
+	return data, nil
+}
diff --git a/internal/flagopt/flagopt.go b/internal/flagopt/flagopt.go
new file mode 100644
index 0000000..484270d
--- /dev/null
+++ b/internal/flagopt/flagopt.go
@@ -0,0 +1,26 @@
+package flagopt
+
+import (
+	"flag"
+	"time"
+)
+
+func BoolOpt(fs *flag.FlagSet, opt *bool, short, long string, value bool) {
+	fs.BoolVar(opt, short, value, "")
+	fs.BoolVar(opt, long, value, "")
+}
+
+func DurationOpt(fs *flag.FlagSet, opt *time.Duration, short, long string, value time.Duration) {
+	fs.DurationVar(opt, short, value, "")
+	fs.DurationVar(opt, long, value, "")
+}
+
+func UintOpt(fs *flag.FlagSet, opt *uint, short, long string, value uint) {
+	fs.UintVar(opt, short, value, "")
+	fs.UintVar(opt, long, value, "")
+}
+
+func StringOpt(fs *flag.FlagSet, opt *string, short, long, value string) {
+	fs.StringVar(opt, short, value, "")
+	fs.StringVar(opt, long, value, "")
+}
diff --git a/internal/ioutil/ioutil.go b/internal/ioutil/ioutil.go
new file mode 100644
index 0000000..7fe6cfc
--- /dev/null
+++ b/internal/ioutil/ioutil.go
@@ -0,0 +1,56 @@
+package ioutil
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+)
+
+func CommitData(path string, data []byte) error {
+	return os.WriteFile(path, data, 0644) // FIXME: use safefile package for atomic file writes
+}
+
+func ReadData(path string) ([]byte, error) {
+	return os.ReadFile(path)
+}
+
+func CommitJSON(path string, obj any) error {
+	b, err := json.MarshalIndent(obj, "", "  ")
+	if err != nil {
+		return err
+	}
+	return CommitData(path, b)
+}
+
+func ReadJSON(path string, obj any) error {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("%s: %v", path, err)
+	}
+	return json.Unmarshal(b, obj)
+}
+
+func CreateDirectories(paths []string) error {
+	for _, path := range paths {
+		if err := os.Mkdir(path, 0755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func DirectoriesExist(paths []string) error {
+	for _, path := range paths {
+		info, err := os.Stat(path)
+		if os.IsNotExist(err) {
+			return fmt.Errorf("directory does not exist: %s", path)
+		}
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			return fmt.Errorf("%s: is not a directory", path)
+		}
+	}
+	return nil
+}
diff --git a/internal/logger/logger.go b/internal/logger/logger.go
new file mode 100644
index 0000000..195ad3e
--- /dev/null
+++ b/internal/logger/logger.go
@@ -0,0 +1,96 @@
+package logger
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"sync"
+)
+
+const (
+	LevelDebug Level = iota + 1
+	LevelInfo
+	LevelNotice
+	LevelFatal
+)
+
+type Level int
+
+func NewLevel(str string) (Level, error) {
+	switch str {
+	case LevelDebug.String():
+		return LevelDebug, nil
+	case LevelInfo.String():
+		return LevelInfo, nil
+	case LevelNotice.String():
+		return LevelNotice, nil
+	case LevelFatal.String():
+		return LevelFatal, nil
+	}
+	return Level(0), fmt.Errorf("unknown level %q", str)
+}
+
+func (lv Level) String() string {
+	switch lv {
+	case LevelDebug:
+		return "DEBUG"
+	case LevelInfo:
+		return "INFO"
+	case LevelNotice:
+		return "NOTICE"
+	case LevelFatal:
+		return "FATAL"
+	default:
+		return "UNKNOWN"
+	}
+}
+
+type Config struct {
+	Level Level
+	File  *os.File
+}
+
+type Logger struct {
+	cfg   Config
+	log   log.Logger
+	mutex sync.Mutex
+}
+
+func New(cfg Config) (l Logger) {
+	l.Reconfigure(cfg)
+	return
+}
+
+func (l *Logger) Reconfigure(cfg Config) {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+
+	if cfg.Level < LevelDebug || cfg.Level > LevelFatal {
+		cfg.Level = LevelNotice
+	}
+	if cfg.File == nil {
+		cfg.File = os.Stdout
+	}
+
+	l.cfg = cfg
+	l.log = *log.New(l.cfg.File, "", log.Ldate|log.Ltime)
+}
+
+func (l *Logger) IsConfigured() bool {
+	return l.cfg.File != nil
+}
+
+func (l *Logger) Debugf(format string, args ...interface{})  { l.printf(LevelDebug, format, args...) }
+func (l *Logger) Infof(format string, args ...interface{})   { l.printf(LevelInfo, format, args...) }
+func (l *Logger) Noticef(format string, args ...interface{}) { l.printf(LevelNotice, format, args...) }
+func (l *Logger) Fatalf(format string, args ...interface{})  { l.printf(LevelFatal, format, args...) }
+func (l *Logger) Dief(format string, args ...interface{})    { l.Fatalf(format, args...); os.Exit(1) }
+
+func (l *Logger) printf(lv Level, format string, args ...interface{}) {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+
+	if l.cfg.Level <= lv {
+		l.log.Printf("["+lv.String()+"] "+format, args...)
+	}
+}
diff --git a/internal/manager/helpers.go b/internal/manager/helpers.go
deleted file mode 100644
index a9a2158..0000000
--- a/internal/manager/helpers.go
+++ /dev/null
@@ -1,52 +0,0 @@
-package manager
-
-import (
-	"crypto/sha256"
-	"encoding/base64"
-	"fmt"
-
-	ct "github.com/google/certificate-transparency-go"
-	"gitlab.torproject.org/rgdd/ct/pkg/metadata"
-	"rgdd.se/silent-ct/pkg/monitor"
-)
-
-func selectLogs(m metadata.Metadata) []monitor.MessageLogConfig {
-	var logs []monitor.MessageLogConfig
-	for _, operator := range m.Operators {
-		for _, log := range operator.Logs {
-			if log.State == nil {
-				continue // ignore logs without a state (should not happen)
-			}
-			if log.State.Name == metadata.LogStatePending {
-				continue // log is not yet relevant
-			}
-			if log.State.Name == metadata.LogStateRetired {
-				continue // log is not expected to be reachable
-			}
-			if log.State.Name == metadata.LogStateRejected {
-				continue // log is not expected to be reachable
-			}
-
-			// FIXME: remove me instead of hard coding Argon 2024
-			id, _ := log.Key.ID()
-			got := fmt.Sprintf("%s", base64.StdEncoding.EncodeToString(id[:]))
-			want := "7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs="
-			if got != want {
-				continue
-			}
-
-			logs = append(logs, monitor.MessageLogConfig{
-				Metadata: log,
-				State: monitor.MonitorState{
-					LogState: monitor.LogState{ct.SignedTreeHead{
-						SHA256RootHash: [sha256.Size]byte{47, 66, 110, 15, 246, 154, 8, 100, 150, 140, 206, 208, 17, 57, 112, 116, 210, 3, 19, 55, 46, 63, 209, 12, 234, 130, 225, 124, 237, 2, 64, 228},
-						TreeSize:       610650601,
-						Timestamp:      1702108968538,
-					}},
-					NextIndex: 388452203,
-				},
-			})
-		}
-	}
-	return logs
-}
diff --git a/internal/manager/manager.go b/internal/manager/manager.go
index 33207e9..bc216c1 100644
--- a/internal/manager/manager.go
+++ b/internal/manager/manager.go
@@ -2,93 +2,208 @@ package manager
 
 import (
 	"context"
-	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"time"
 
 	"gitlab.torproject.org/rgdd/ct/pkg/metadata"
-	"rgdd.se/silent-ct/pkg/monitor"
-	"rgdd.se/silent-ct/pkg/server"
-)
-
-const (
-	DefaultStateDir                = "/home/rgdd/.local/share/silent-ct" // FIXME
-	DefaultMetadataRefreshInterval = 1 * time.Hour
+	"rgdd.se/silent-ct/internal/feedback"
+	"rgdd.se/silent-ct/internal/logger"
+	"rgdd.se/silent-ct/internal/monitor"
+	"rgdd.se/silent-ct/pkg/policy"
+	"rgdd.se/silent-ct/pkg/storage"
 )
 
 type Config struct {
-	StateDir string
-	Nodes    server.Nodes
+	Policy    policy.Policy
+	Bootstrap bool   // Whether a new storage should be initialized from scratch
+	Directory string // Path to a directory where everything will be stored
 
-	MetadataRefreshInterval time.Duration
+	// Optional
+	Logger                  logger.Logger // Where to output messages and with what verbosity
+	AlertDelay              time.Duration // Time before alerting on certificates that are unaccounted for
+	MetadataRefreshInterval time.Duration // How often to update the list of monitored logs
+	ShutdownTimeout         time.Duration // Force shutdown after this timeout (FIXME: should not be needed)
 }
 
 type Manager struct {
 	Config
+	storage.Storage
+
+	feventCh  chan []feedback.Event
+	meventCh  chan monitor.Event
+	mconfigCh chan monitor.MonitoredLog
+	errorCh   chan error
 }
 
-func New(cfg Config) (Manager, error) {
-	if cfg.StateDir == "" {
-		cfg.StateDir = DefaultStateDir
+func New(cfg Config, fch chan []feedback.Event, mch chan monitor.Event, cch chan monitor.MonitoredLog, ech chan error) (Manager, error) {
+	if !cfg.Logger.IsConfigured() {
+		cfg.Logger = logger.New(logger.Config{Level: logger.LevelNotice, File: os.Stdout})
 	}
 	if cfg.MetadataRefreshInterval == 0 {
-		cfg.MetadataRefreshInterval = DefaultMetadataRefreshInterval
+		cfg.MetadataRefreshInterval = 1 * time.Hour
+	}
+	if cfg.ShutdownTimeout == 0 {
+		cfg.ShutdownTimeout = 1 * time.Second // FIXME: increase
 	}
-	return Manager{Config: cfg}, nil
-}
-
-func (mgr *Manager) Run(ctx context.Context,
-	serverCh chan server.MessageNodeSubmission,
-	monitorCh chan monitor.MessageLogProgress,
-	configCh chan []monitor.MessageLogConfig,
-	errorCh chan error) error {
 
-	md, err := mgr.metadataRead()
+	s, err := storage.New(storage.Config{
+		Bootstrap:  cfg.Bootstrap,
+		Directory:  cfg.Directory,
+		AlertDelay: cfg.AlertDelay,
+		StaticLogs: cfg.Policy.StaticLogs,
+		RemoveLogs: cfg.Policy.RemoveLogs,
+	})
 	if err != nil {
-		return fmt.Errorf("read metadata: %v\n", err)
+		return Manager{}, err
+	}
+
+	for _, log := range s.LogList.Generate() {
+		state, err := s.BootstrapLog(context.Background(), log, cfg.Bootstrap)
+		if errors.Is(err, storage.ErrorMonitorStateExists) {
+			continue
+		}
+		if err != nil {
+			return Manager{}, err
+		}
+		cfg.Logger.Infof("bootstrapping log %s at next index %d\n", log.URL, state.NextIndex)
+	}
+
+	return Manager{Config: cfg, Storage: s, feventCh: fch, meventCh: mch, mconfigCh: cch, errorCh: ech}, nil
+}
+
+func (mgr *Manager) Run(ctx context.Context) error {
+	if err := mgr.startupConfig(); err != nil {
+		return fmt.Errorf("unable to do startup config: %v", err)
 	}
-	configCh <- selectLogs(md)
 
-	ticker := time.NewTicker(mgr.MetadataRefreshInterval)
-	defer ticker.Stop()
+	metadataTicker := time.NewTicker(mgr.MetadataRefreshInterval)
+	defer metadataTicker.Stop()
 
+	shutdown := false
 	for {
 		select {
-		case <-ctx.Done():
-			return nil
-		case <-ticker.C:
-			mu, err := mgr.metadataUpdate(ctx, md)
-			if err != nil {
-				continue
+		case <-metadataTicker.C:
+			if err := mgr.metadataJob(ctx); err != nil {
+				return fmt.Errorf("unable to run metadata job: %v", err)
+			}
+		case ev := <-mgr.meventCh:
+			if err := mgr.monitorJob(ev); err != nil {
+				return fmt.Errorf("unable to run monitor job: %v", err)
+			}
+			if err := mgr.alertJob(); err != nil {
+				return fmt.Errorf("unable to run alert job: %v\n", err)
 			}
-			if mu.Version.Major <= md.Version.Major {
+		case ev := <-mgr.feventCh:
+			if err := mgr.feedbackJob(ev); err != nil {
+				return fmt.Errorf("unable to run server job: %v", err)
+			}
+			if err := mgr.alertJob(); err != nil {
+				return fmt.Errorf("unable to run alert job: %v\n", err)
+			}
+		case err := <-mgr.errorCh:
+			if err := mgr.errorJob(err); err != nil {
+				return fmt.Errorf("unable to run error job: %v", err)
+			}
+		case <-ctx.Done():
+			if !shutdown {
+				shutdown = true
+				mgr.Logger.Noticef("shutdown scheduled in %v\n", mgr.ShutdownTimeout)
+
+				// defer shutdown so that all channels can be drained
+				var cancel context.CancelFunc
+				ctx, cancel = context.WithTimeout(context.Background(), mgr.ShutdownTimeout)
+				defer cancel()
 				continue
 			}
-			md = mu
-			configCh <- selectLogs(md)
-		case ev := <-monitorCh:
-			fmt.Printf("DEBUG: received event from monitor with %d matches\n", len(ev.Matches))
-		case ev := <-serverCh:
-			fmt.Printf("DEBUG: received event from server: %v\n", ev)
-		case err := <-errorCh:
-			fmt.Printf("DEBUG: received error: %v\n", err)
+
+			mgr.Logger.Debugf("manager shutdown\n")
+			os.Exit(0) // FIXME: return nil without hanging, unpredictable gh.com/google/ct-go fetcher shutdown?
 		}
 	}
 }
 
-func (mgr *Manager) metadataRead() (metadata.Metadata, error) {
-	b, err := os.ReadFile(mgr.StateDir + "/metadata.json")
+func (mgr *Manager) startupConfig() error {
+	mgr.Logger.Debugf("startup configuring contains %d logs\n", len(mgr.Storage.LogList.Generate()))
+	for _, log := range mgr.Storage.LogList.Generate() {
+		state, err := mgr.GetMonitorState(log)
+		if err != nil {
+			return err
+		}
+		mgr.mconfigCh <- monitor.MonitoredLog{Config: log, State: state}
+	}
+	return nil
+}
+
+func (mgr *Manager) metadataJob(ctx context.Context) error {
+	mgr.Logger.Debugf("running metadata job\n")
+	added, removed, err := mgr.LogList.Update(ctx)
+	if err != nil {
+		if mgr.LogList.IsStale() {
+			return fmt.Errorf("unable to update log list which is now stale: %v", err)
+		}
+	}
+
+	mgr.removeLogs(removed)
+	mgr.addLogs(ctx, added)
+	return nil
+}
+
+func (mgr *Manager) removeLogs(logs []metadata.Log) {
+	mgr.Logger.Debugf("removing %d logs\n", len(logs))
+	for _, log := range logs {
+		state, _ := mgr.GetMonitorState(log)
+		mgr.Logger.Infof("removing log %s with %d entries in its backlog\n", log.URL, state.TreeSize-state.NextIndex)
+		mgr.mconfigCh <- monitor.MonitoredLog{Config: log}
+	}
+}
+
+func (mgr *Manager) addLogs(ctx context.Context, logs []metadata.Log) {
+	mgr.Logger.Debugf("adding %d logs\n", len(logs))
+	for _, log := range logs {
+		state, err := mgr.BootstrapLog(ctx, log, false)
+		if errors.Is(err, storage.ErrorMonitorStateExists) {
+			mgr.Logger.Infof("adding log %s with existing state on disk\n", log.URL)
+		} else if err != nil {
+			mgr.Logger.Noticef("restart required: failed to bootstrap new log %s: %v\n", log.URL, err)
+		} else {
+			mgr.Logger.Infof("bootstrapping log %s at next index 0\n", log.URL)
+		}
+		mgr.mconfigCh <- monitor.MonitoredLog{Config: log, State: state}
+	}
+}
+
+func (mgr *Manager) feedbackJob(events []feedback.Event) error {
+	mgr.Logger.Debugf("received feedback with %d events", len(events))
+	for _, ev := range events {
+		if err := mgr.AddChain(ev.NodeName, ev.PEMChain); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (mgr *Manager) monitorJob(msg monitor.Event) error {
+	mgr.Logger.Debugf("new state for %s\n", msg.Summary())
+	if err := mgr.AddEntries(msg.State.LogID, msg.Matches); err != nil {
+		return err
+	}
+	return mgr.SetMonitorState(msg.State.LogID, msg.State)
+}
+
+func (mgr *Manager) alertJob() error {
+	alerts, err := mgr.Index.TriggerAlerts()
 	if err != nil {
-		return metadata.Metadata{}, err
+		return err
 	}
-	var md metadata.Metadata
-	if err := json.Unmarshal(b, &md); err != nil {
-		return metadata.Metadata{}, err
+	for _, alert := range alerts {
+		mgr.Logger.Noticef("certificate mis-issuance? No node submitted certificate %s\n", alert.StoredAt)
 	}
-	return md, nil
+	return nil
 }
 
-func (mgr *Manager) metadataUpdate(ctx context.Context, old metadata.Metadata) (metadata.Metadata, error) {
-	return metadata.Metadata{}, fmt.Errorf("TODO: update metadata")
+func (mgr *Manager) errorJob(err error) error {
+	mgr.Logger.Debugf("received error: %v\n", err)
+	return nil
 }
diff --git a/internal/merkle/TODO b/internal/merkle/TODO
deleted file mode 100644
index 46cc0cb..0000000
--- a/internal/merkle/TODO
+++ /dev/null
@@ -1 +0,0 @@
-Drop this package, fix the minor edit in upstream.
diff --git a/internal/merkle/compact.go b/internal/merkle/compact.go
deleted file mode 100644
index 6eeabd0..0000000
--- a/internal/merkle/compact.go
+++ /dev/null
@@ -1,115 +0,0 @@
-// BSD 2-Clause License
-//
-// Copyright (c) 2022, the ct authors
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//  1. Redistributions of source code must retain the above copyright notice, this
-//     list of conditions and the following disclaimer.
-//
-//  2. Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// From:
-// https://gitlab.torproject.org/rgdd/ct/-/tree/main/pkg/merkle
-//
-// The only difference is that leaf hashes rather than leaf data are passed as
-// input to TreeHeadFromRangeProof, thus also changing the nodes() helper.
-package merkle
-
-import (
-	"crypto/sha256"
-	"fmt"
-)
-
-// node represents a subtree at some level and a particular index
-type node struct {
-	index uint64
-	hash  [sha256.Size]byte
-}
-
-// nodes returns a list of consecutive leaf hashes
-func nodes(index uint64, leafHashes [][sha256.Size]byte) (n []node) {
-	for i, lh := range leafHashes {
-		n = append(n, node{index + uint64(i), lh})
-	}
-	return
-}
-
-// compactRange outputs the minimal number of fixed subtree hashes given a
-// non-empty list of consecutive leaves that start from a non-zero index.  For a
-// definition of this algorithm, see the end of ../../doc/tlog_algorithms.md.
-func compactRange(nodes []node) [][sha256.Size]byte {
-	// Step 1
-	var hashes [][sha256.Size]byte
-
-	// Step 2
-	for len(nodes) > 1 {
-		// Step 2a
-		if xor(nodes[1].index, 1) != nodes[0].index {
-			hashes = append(hashes, nodes[0].hash)
-			nodes = nodes[1:]
-		}
-
-		// Step 2b; Step 2c;        Step 2c(iii)
-		for i := 0; i < len(nodes); i++ {
-			// Step 2c(i)
-			if i+1 != len(nodes) {
-				nodes[i].hash = HashInteriorNode(nodes[i].hash, nodes[i+1].hash)
-				nodes = append(nodes[:i+1], nodes[i+2:]...)
-			}
-
-			// Step 2c(ii)
-			nodes[i].index = rshift(nodes[i].index)
-		}
-	}
-
-	// Step 3
-	return append(hashes, nodes[0].hash)
-}
-
-// TreeHeadFromRangeProof computes a tree head at size n=len(leafHashes)+index
-// if given a list of leaf hashes at indices index,...,n-1 as well as an
-// inclusion proof for the first leaf in the tree of size n.  This allows a
-// verifier to check inclusion of one or more log entries with a single
-// inclusion proof.
-func TreeHeadFromRangeProof(leafHashes [][sha256.Size]byte, index uint64, proof [][sha256.Size]byte) (root [sha256.Size]byte, err error) {
-	var cr [][sha256.Size]byte
-	confirmHash := func(h [sha256.Size]byte) error {
-		if h != cr[0] {
-			return fmt.Errorf("aborted due incorrect right-node subtree hash")
-		}
-		cr = cr[1:]
-		return nil
-	}
-	copyRoot := func(r [sha256.Size]byte) error {
-		root = r
-		return nil
-	}
-
-	if len(leafHashes) == 0 {
-		return [sha256.Size]byte{}, fmt.Errorf("need at least one leaf to recompute tree head from proof")
-	}
-	if len(leafHashes) > 1 {
-		cr = compactRange(nodes(index+1, leafHashes[1:]))
-	}
-	return root, inclusion(leafHashes[0], index, index+uint64(len(leafHashes)), proof, copyRoot, confirmHash)
-}
-
-func xor(a, b uint64) uint64 {
-	return a ^ b
-}
diff --git a/internal/merkle/merkle.go b/internal/merkle/merkle.go
deleted file mode 100644
index 872364f..0000000
--- a/internal/merkle/merkle.go
+++ /dev/null
@@ -1,271 +0,0 @@
-// BSD 2-Clause License
-//
-// Copyright (c) 2022, the ct authors
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//  1. Redistributions of source code must retain the above copyright notice, this
-//     list of conditions and the following disclaimer.
-//
-//  2. Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// From:
-// https://gitlab.torproject.org/rgdd/ct/-/tree/main/pkg/merkle
-package merkle
-
-import (
-	"crypto/sha256"
-	"fmt"
-)
-
-// HashEmptyTree computes the hash of an empty tree.  See RFC 6162, §2.1:
-//
-//	MTH({}) = SHA-256()
-func HashEmptyTree() [sha256.Size]byte {
-	return sha256.Sum256(nil)
-}
-
-// HashLeafNode computes the hash of a leaf's data.  See RFC 6162, §2.1:
-//
-//	MTH({d(0)}) = SHA-256(0x00 || d(0))
-func HashLeafNode(data []byte) (hash [sha256.Size]byte) {
-	h := sha256.New()
-	h.Write([]byte{0x00})
-	h.Write(data)
-	copy(hash[:], h.Sum(nil))
-	return
-}
-
-// HashInteriorNode computes the hash of an interior node.  See RFC 6962, §2.1:
-//
-//	MTH(D[n]) = SHA-256(0x01 || MTH(D[0:k]) || MTH(D[k:n])
-func HashInteriorNode(left, right [sha256.Size]byte) (hash [sha256.Size]byte) {
-	h := sha256.New()
-	h.Write([]byte{0x01})
-	h.Write(left[:])
-	h.Write(right[:])
-	copy(hash[:], h.Sum(nil))
-	return
-}
-
-// inclusion implements the algorithm specified in RFC 9162, Section 2.1.3.2.
-// In addition, the caller is allowed to confirm right-node subtree hashes.
-func inclusion(leaf [sha256.Size]byte, index, size uint64, proof [][sha256.Size]byte,
-	confirmRoot func([sha256.Size]byte) error, confirmHash func([sha256.Size]byte) error) error {
-	// Step 1
-	if index >= size {
-		return fmt.Errorf("leaf index must be in [%d, %d]", 0, size-1)
-	}
-
-	// Step 2
-	fn := index
-	sn := size - 1
-
-	// Step 3
-	r := leaf
-
-	// Step 4
-	for i, p := range proof {
-		// Step 4a
-		if sn == 0 {
-			return fmt.Errorf("reached tree head with %d remaining proof hash(es)", len(proof[i:]))
-		}
-
-		// Step 4b
-		if isLSB(fn) || fn == sn {
-			// Step 4b, i
-			r = HashInteriorNode(p, r)
-
-			// Step 4b, ii
-			if !isLSB(fn) {
-				for {
-					fn = rshift(fn)
-					sn = rshift(sn)
-
-					if isLSB(fn) || fn == 0 {
-						break
-					}
-				}
-			}
-		} else {
-			// Step 4b, i
-			r = HashInteriorNode(r, p)
-
-			// Extension: allow the caller to confirm right-node subtree hashes
-			if err := confirmHash(p); err != nil {
-				return fmt.Errorf("subtree index %d: %v", fn, err)
-			}
-		}
-
-		// Step 4c
-		fn = rshift(fn)
-		sn = rshift(sn)
-	}
-
-	// Step 5
-	if sn != 0 {
-		return fmt.Errorf("stopped at subtree with index %d due to missing proof hashes", fn)
-	}
-	return confirmRoot(r)
-}
-
-// consistency implements the algorithm specified in RFC 9162, §2.1.4.2
-func consistency(oldSize, newSize uint64, oldRoot, newRoot [sha256.Size]byte, proof [][sha256.Size]byte) error {
-	// Step 1
-	if len(proof) == 0 {
-		return fmt.Errorf("need at least one proof hash")
-	}
-
-	// Step 2
-	if isPOW2(oldSize) {
-		proof = append([][sha256.Size]byte{oldRoot}, proof...)
-	}
-
-	// Step 3
-	fn := oldSize - 1
-	sn := newSize - 1
-
-	// Step 4
-	for isLSB(fn) {
-		fn = rshift(fn)
-		sn = rshift(sn)
-	}
-
-	// Step 5
-	fr := proof[0]
-	sr := proof[0]
-
-	// Step 6
-	for i, c := range proof[1:] {
-		// Step 6a
-		if sn == 0 {
-			return fmt.Errorf("reached tree head with %d remaining proof hash(es)", len(proof[i+1:]))
-		}
-
-		// Step 6b
-		if isLSB(fn) || fn == sn {
-			// Step 6b, i
-			fr = HashInteriorNode(c, fr)
-			// Step 6b, ii
-			sr = HashInteriorNode(c, sr)
-			// Step 6b, iii
-			if !isLSB(fn) {
-				for {
-					fn = rshift(fn)
-					sn = rshift(sn)
-
-					if isLSB(fn) || fn == 0 {
-						break
-					}
-				}
-			}
-		} else {
-			// Step 6b, i
-			sr = HashInteriorNode(sr, c)
-		}
-
-		// Step 6c
-		fn = rshift(fn)
-		sn = rshift(sn)
-	}
-
-	// Step 7
-	if sn != 0 {
-		return fmt.Errorf("stopped at subtree with index %d due to missing proof hashes", fn)
-	}
-	if fr != oldRoot {
-		return fmt.Errorf("recomputed old tree head %x is not equal to reference tree head %x", fr[:], oldRoot[:])
-	}
-	if sr != newRoot {
-		return fmt.Errorf("recomputed new tree head %x is not equal to reference tree head %x", sr[:], newRoot[:])
-	}
-	return nil
-}
-
-// VerifyInclusion verifies that a leaf's data is commited at a given index in a
-// reference tree
-func VerifyInclusion(data []byte, index, size uint64, root [sha256.Size]byte, proof [][sha256.Size]byte) error {
-	if size == 0 {
-		return fmt.Errorf("tree size must be larger than zero")
-	}
-
-	confirmHash := func(h [sha256.Size]byte) error { return nil } // No compact range extension
-	confirmRoot := func(r [sha256.Size]byte) error {
-		if r != root {
-			return fmt.Errorf("recomputed tree head %x is not equal to reference tree head %x", r[:], root[:])
-		}
-		return nil
-	}
-	return inclusion(HashLeafNode(data), index, size, proof, confirmRoot, confirmHash)
-}
-
-// VerifyConsistency verifies that an an old tree is consistent with a new tree
-func VerifyConsistency(oldSize, newSize uint64, oldRoot, newRoot [sha256.Size]byte, proof [][sha256.Size]byte) error {
-	checkTree := func(size uint64, root [sha256.Size]byte) error {
-		if size == 0 {
-			if root != HashEmptyTree() {
-				return fmt.Errorf("non-empty tree head %x for size zero", root[:])
-			}
-			if len(proof) != 0 {
-				return fmt.Errorf("non-empty proof with %d hashes for size zero", len(proof))
-			}
-		} else if root == HashEmptyTree() {
-			return fmt.Errorf("empty tree head %x for tree size %d", root[:], size)
-		}
-		return nil
-	}
-
-	if err := checkTree(oldSize, oldRoot); err != nil {
-		return fmt.Errorf("old: %v", err)
-	}
-	if err := checkTree(newSize, newRoot); err != nil {
-		return fmt.Errorf("new: %v", err)
-	}
-	if oldSize == 0 {
-		return nil
-	}
-
-	if oldSize == newSize {
-		if oldRoot != newRoot {
-			return fmt.Errorf("different tree heads %x and %x with equal tree size %d", oldRoot, newRoot, oldSize)
-		}
-		if len(proof) != 0 {
-			return fmt.Errorf("non-empty proof with %d hashes for equal tree size %d", len(proof), oldSize)
-		}
-		return nil
-	}
-	if oldSize > newSize {
-		return fmt.Errorf("old tree size %d must be smaller than or equal to the new tree size %d", oldSize, newSize)
-	}
-
-	return consistency(oldSize, newSize, oldRoot, newRoot, proof)
-}
-
-// isLSB returns true if the least significant bit of num is set
-func isLSB(num uint64) bool {
-	return (num & 1) != 0
-}
-
-// isPOW2 returns true if num is a power of two (1, 2, 4, 8, ...)
-func isPOW2(num uint64) bool {
-	return (num & (num - 1)) == 0
-}
-
-func rshift(num uint64) uint64 {
-	return num >> 1
-}
diff --git a/internal/monitor/chunks.go b/internal/monitor/chunks.go
new file mode 100644
index 0000000..02b3802
--- /dev/null
+++ b/internal/monitor/chunks.go
@@ -0,0 +1,89 @@
+package monitor
+
+//
+// A min heap of chunks, ordered on each chunk's start index.  This makes it
+// easy to order the downloaded leaves when using multiple parallell fetchers.
+//
+// Credit: inspiration to use a heap from Aaron Gable, see
+// https://github.com/aarongable/ctaudit
+//
+
+import (
+	"container/heap"
+	"crypto/sha256"
+)
+
+type chunk struct {
+	startIndex uint64              // Index of the first leaf
+	leafHashes [][sha256.Size]byte // List of consecutive leaf hashes
+	matches    []LogEntry          // Leaves that match some criteria
+	errors     []error             // Errors that ocurred while matching on the leaves
+}
+
+type chunks []*chunk
+
+func newChunks() *chunks {
+	var h chunks
+	heap.Init((*internal)(&h))
+	return &h
+}
+
+func (h *chunks) push(c *chunk) {
+	heap.Push((*internal)(h), c)
+}
+
+func (h *chunks) pop() *chunk {
+	x := heap.Pop((*internal)(h))
+	return x.(*chunk)
+}
+
+// gap returns true if there's a gap between the provided start index and the
+// top most chunk.  If the top most chunk is in sequence, it is merged with
+// any following chunks that are also in sequence to form one larger chunk.
+func (h *chunks) gap(start uint64) bool {
+	if len(*h) == 0 {
+		return true
+	}
+
+	top := h.pop()
+	if start != top.startIndex {
+		h.push(top)
+		return true
+	}
+
+	for len(*h) > 0 {
+		c := h.pop()
+		if c.startIndex != top.startIndex+uint64(len(top.leafHashes)) {
+			h.push(c)
+			break
+		}
+
+		top.leafHashes = append(top.leafHashes, c.leafHashes...)
+		top.matches = append(top.matches, c.matches...)
+		top.errors = append(top.errors, c.errors...)
+	}
+
+	h.push(top)
+	return false
+}
+
+// internal implements the heap interface, see example:
+// https://cs.opensource.google/go/go/+/refs/tags/go1.21.5:src/container/heap/example_intheap_test.go
+type internal chunks
+
+func (h internal) Len() int           { return len(h) }
+func (h internal) Less(i, j int) bool { return h[i].startIndex < h[j].startIndex }
+func (h internal) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *internal) Push(x any) {
+	*h = append(*h, x.(*chunk))
+}
+
+func (h *internal) Pop() any {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	old[n-1] = nil // avoid memory leak
+	*h = old[:n-1]
+	return x
+}
diff --git a/internal/monitor/matcher.go b/internal/monitor/matcher.go
new file mode 100644
index 0000000..912e595
--- /dev/null
+++ b/internal/monitor/matcher.go
@@ -0,0 +1,13 @@
+package monitor
+
+type Matcher interface {
+	// Match determines if a log entry is considered to be a "match" based on
+	// some criteria.  An error is returned if any certificate parsing fails.
+	Match(leafInput, extraData []byte) (bool, error)
+}
+
+type MatchAll struct{}
+
+func (m *MatchAll) Match(leafInput, extraData []byte) (bool, error) {
+	return true, nil
+}
diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go
new file mode 100644
index 0000000..6accd97
--- /dev/null
+++ b/internal/monitor/monitor.go
@@ -0,0 +1,173 @@
+// Package monitor provides monitoring of Certificate Transparency logs.  If
+// running in continuous mode, the list of logs can be updated dynamically.
+//
+// Implement the Matcher interface to customize which certificates should be
+// included in the monitor's emitted events.  See MatchAll for an example.
+//
+// Note that this package verifies that the monitored logs are locally
+// consistent with regard to the initial start-up state.  It is up to the user
+// to process the monitor's emitted events and errors, and to persist state.
+package monitor
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+
+	ct "github.com/google/certificate-transparency-go"
+	"github.com/google/certificate-transparency-go/client"
+	"github.com/google/certificate-transparency-go/jsonclient"
+	"gitlab.torproject.org/rgdd/ct/pkg/metadata"
+	"rgdd.se/silent-ct/internal/logger"
+)
+
+// MonitoredLog provides information about a log the monitor is following
+type MonitoredLog struct {
+	Config metadata.Log
+	State  State
+}
+
+// State is the latest append-only state the monitor observed from its local
+// vantage point.  The next entry to download is specified by NextIndex.
+type State struct {
+	ct.SignedTreeHead `json:"latest_sth"`
+	NextIndex         uint64 `json:"next_index"`
+}
+
+// Event carries the latest consistent monitor state, found matches, as well as
+// errors that occurred while trying to match on the downloaded log entries.
+type Event struct {
+	State   State
+	Matches []LogEntry
+	Errors  []error
+}
+
+func (ev *Event) Summary() string {
+	return fmt.Sprintf("log %s: tree size %d at next index %d (%d matches, %d errors)",
+		base64.StdEncoding.EncodeToString(ev.State.LogID[:]),
+		ev.State.TreeSize, ev.State.NextIndex, len(ev.Matches), len(ev.Errors))
+}
+
+// LogEntry is a Merkle tree leaf in a log
+type LogEntry struct {
+	LeafIndex uint64 `json:"leaf_index"`
+	LeafData  []byte `json:"leaf_data"`
+	ExtraData []byte `json:"extra_data"`
+}
+
+type Config struct {
+	// Optional
+	Matcher    Matcher       // Which log entries to match (default is to match all)
+	Logger     logger.Logger // Debug prints only (no output by default)
+	Contact    string        // Something that help log operators get in touch
+	ChunkSize  uint          // Min number of leaves to propagate a chunk without matches
+	BatchSize  uint          // Max number of certificates to accept per worker
+	NumWorkers uint          // Number of parallel workers to use for each log
+}
+
+type Monitor struct {
+	cfg     Config
+	matcher Matcher
+
+	eventCh  chan Event
+	configCh chan MonitoredLog
+	errorCh  chan error
+}
+
+func New(cfg Config, evCh chan Event, cfgCh chan MonitoredLog, errCh chan error) (Monitor, error) {
+	if cfg.Matcher == nil {
+		cfg.Matcher = &MatchAll{}
+	}
+	if !cfg.Logger.IsConfigured() {
+		cfg.Logger = logger.New(logger.Config{Level: logger.LevelNotice, File: os.Stderr})
+	}
+	if cfg.Contact == "" {
+		cfg.Contact = "unknown-user"
+	}
+	if cfg.ChunkSize == 0 {
+		cfg.ChunkSize = 256 // FIXME: 16364
+	}
+	if cfg.BatchSize == 0 {
+		cfg.BatchSize = 1024
+	}
+	if cfg.NumWorkers == 0 {
+		cfg.NumWorkers = 2
+	}
+	return Monitor{cfg: cfg, matcher: cfg.Matcher, eventCh: evCh, configCh: cfgCh, errorCh: errCh}, nil
+}
+
+func (mon *Monitor) RunOnce(ctx context.Context, cfg []MonitoredLog, evCh chan Event, errCh chan error) error {
+	return fmt.Errorf("TODO")
+}
+
+func (mon *Monitor) RunForever(ctx context.Context) error {
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	mctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	monitoring := make(map[metadata.LogURL]context.CancelFunc)
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case log := <-mon.configCh:
+			if tcancel, ok := monitoring[log.Config.URL]; ok {
+				delete(monitoring, log.Config.URL)
+				tcancel()
+				continue
+			}
+
+			newTail := mon.newTailRFC6962
+			if log.Config.DNS != nil { // FIXME: get a real nob for tile-based logs
+				newTail = mon.newTailTile
+			}
+			t, err := newTail(log)
+			if err != nil {
+				return err
+			}
+
+			tctx, tcancel := context.WithCancel(mctx)
+			monitoring[log.Config.URL] = tcancel
+
+			wg.Add(1)
+			go func(log MonitoredLog, t tail) {
+				defer wg.Done()
+				defer tcancel()
+				t.run(tctx, log, mon.eventCh, mon.errorCh)
+			}(log, t)
+		}
+	}
+}
+
+const userAgentPrefix = "rgdd.se/silent-ct"
+
+func (mon *Monitor) newTailRFC6962(log MonitoredLog) (tail, error) {
+	key, err := x509.MarshalPKIXPublicKey(log.Config.Key.Public)
+	if err != nil {
+		return tail{}, err
+	}
+	cli, err := client.New(string(log.Config.URL), &http.Client{}, jsonclient.Options{
+		Logger:       &discard{},
+		UserAgent:    userAgentPrefix + "/" + mon.cfg.Contact,
+		PublicKeyDER: key,
+	})
+	if err != nil {
+		return tail{}, err
+	}
+
+	return tail{cfg: mon.cfg, scanner: cli, checker: cli, matcher: mon.matcher}, nil
+}
+
+func (mon *Monitor) newTailTile(cfg MonitoredLog) (tail, error) {
+	return tail{}, fmt.Errorf("TODO")
+}
+
+type discard struct{}
+
+func (n *discard) Printf(string, ...interface{}) {}
diff --git a/internal/monitor/tail.go b/internal/monitor/tail.go
new file mode 100644
index 0000000..0e16476
--- /dev/null
+++ b/internal/monitor/tail.go
@@ -0,0 +1,200 @@
+package monitor
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"sync"
+	"time"
+
+	ct "github.com/google/certificate-transparency-go"
+	"github.com/google/certificate-transparency-go/client"
+	"github.com/google/certificate-transparency-go/scanner"
+	"gitlab.torproject.org/rgdd/ct/pkg/merkle"
+)
+
+type tail struct {
+	cfg     Config
+	matcher Matcher
+	scanner scanner.LogClient
+	checker client.CheckLogClient
+}
+
+func (t *tail) run(ctx context.Context, mon MonitoredLog, eventCh chan Event, errorCh chan error) {
+	chunkCh := make(chan *chunk)
+	defer close(chunkCh)
+
+	mctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	callback := func(eb scanner.EntryBatch) {
+		c := chunk{startIndex: uint64(eb.Start)}
+		for i := 0; i < len(eb.Entries); i++ {
+			c.leafHashes = append(c.leafHashes, merkle.HashLeafNode(eb.Entries[i].LeafInput))
+			match, err := t.matcher.Match(eb.Entries[i].LeafInput, eb.Entries[i].ExtraData)
+			if err != nil {
+				c.errors = append(c.errors, fmt.Errorf("while processing index %d for %s: %v", i, mon.Config.URL, err))
+				continue
+			}
+			if !match {
+				continue
+			}
+
+			c.matches = append(c.matches, LogEntry{
+				LeafIndex: c.startIndex + uint64(i),
+				LeafData:  eb.Entries[i].LeafInput,
+				ExtraData: eb.Entries[i].ExtraData,
+			})
+		}
+
+		chunkCh <- &c
+	}
+
+	fetcher := scanner.NewFetcher(t.scanner, &scanner.FetcherOptions{
+		BatchSize:     int(t.cfg.BatchSize),
+		StartIndex:    int64(mon.State.NextIndex),
+		ParallelFetch: int(t.cfg.NumWorkers),
+		Continuous:    true, // FIXME: don't set this for read-only log
+	})
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer cancel()
+		fetcher.Run(mctx, callback)
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer cancel()
+		t.sequence(mctx, mon, eventCh, errorCh, chunkCh)
+	}()
+}
+
+func (t *tail) sequence(ctx context.Context, mon MonitoredLog, eventCh chan Event, errorCh chan error, chunkCh chan *chunk) {
+	state := mon.State
+	heap := newChunks()
+	for {
+		select {
+		case <-ctx.Done():
+			return // FIXME: check if we can pop something before return
+		case c := <-chunkCh:
+			heap.push(c)
+			if heap.gap(state.NextIndex) {
+				continue
+			}
+			c = heap.pop()
+			if len(c.matches) == 0 && len(c.leafHashes) < int(t.cfg.ChunkSize) {
+				heap.push(c)
+				continue // FIXME: don't trigger if we havn't run nextState for too long
+			}
+			nextState, err := t.nextState(ctx, state, c)
+			if err != nil {
+				errorCh <- err
+				heap.push(c)
+				continue
+			}
+
+			state = nextState
+			eventCh <- Event{State: state, Matches: c.matches, Errors: c.errors}
+		}
+	}
+}
+
+func (t *tail) nextState(ctx context.Context, state State, c *chunk) (State, error) {
+	newState, err := t.nextConsistentState(ctx, state)
+	if err != nil {
+		return State{}, err
+	}
+	newState, err = t.nextIncludedState(ctx, newState, c)
+	if err != nil {
+		return State{}, err
+	}
+	return newState, nil
+}
+
+func (t *tail) nextConsistentState(ctx context.Context, state State) (State, error) {
+	sth, err := getSignedTreeHead(ctx, t.checker)
+	if err != nil {
+		return State{}, fmt.Errorf("%s: get-sth: %v", t.checker.BaseURI(), err)
+	}
+	sth.LogID = state.SignedTreeHead.LogID
+	oldSize := state.TreeSize
+	oldRoot := state.SHA256RootHash
+	newSize := sth.TreeSize
+	newRoot := sth.SHA256RootHash
+
+	proof, err := getConsistencyProof(ctx, t.checker, oldSize, newSize)
+	if err != nil {
+		return State{}, fmt.Errorf("%s: get-consistency: %v", t.checker.BaseURI(), err)
+	}
+	if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(proof)); err != nil {
+		return State{}, fmt.Errorf("%s: verify consistency: %v", t.checker.BaseURI(), err)
+	}
+
+	return State{SignedTreeHead: *sth, NextIndex: state.NextIndex}, nil
+}
+
+func (t *tail) nextIncludedState(ctx context.Context, state State, c *chunk) (State, error) {
+	leafHash := c.leafHashes[0]
+	oldSize := state.NextIndex + uint64(len(c.leafHashes))
+	iproof, err := getInclusionProof(ctx, t.checker, leafHash, oldSize) // FIXME: set leaf index in ctx to hack into tile API
+	if err != nil {
+		return State{}, fmt.Errorf("%s: get-inclusion: %v", t.checker.BaseURI(), err)
+	}
+	if got, want := uint64(iproof.LeafIndex), state.NextIndex; got != want {
+		return State{}, fmt.Errorf("%s: wrong index for get-inclusion proof query %x:%d", t.checker.BaseURI(), leafHash[:], oldSize)
+	}
+	oldRoot, err := merkle.TreeHeadFromRangeProof(c.leafHashes, state.NextIndex, unslice(iproof.AuditPath))
+	if err != nil {
+		return State{}, fmt.Errorf("%s: range proof: %v", t.checker.BaseURI(), err)
+	}
+
+	newSize := state.TreeSize
+	newRoot := state.SHA256RootHash
+	cproof, err := getConsistencyProof(ctx, t.checker, oldSize, newSize)
+	if err != nil {
+		return State{}, fmt.Errorf("%s: get-consistency: %v", t.checker.BaseURI(), err)
+	}
+	if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(cproof)); err != nil {
+		return State{}, fmt.Errorf("%s: verify consistency: %v", t.checker.BaseURI(), err)
+	}
+
+	state.NextIndex += uint64(len(c.leafHashes))
+	return state, nil
+}
+
+func getInclusionProof(ctx context.Context, cli client.CheckLogClient, leafHash [sha256.Size]byte, size uint64) (*ct.GetProofByHashResponse, error) {
+	rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	return cli.GetProofByHash(rctx, leafHash[:], size)
+}
+
+func getConsistencyProof(ctx context.Context, cli client.CheckLogClient, oldSize, newSize uint64) ([][]byte, error) {
+	if oldSize == 0 || oldSize >= newSize {
+		return [][]byte{}, nil
+	}
+	rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	return cli.GetSTHConsistency(rctx, oldSize, newSize)
+}
+
+func getSignedTreeHead(ctx context.Context, cli client.CheckLogClient) (*ct.SignedTreeHead, error) {
+	rctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	return cli.GetSTH(rctx)
+}
+
+func unslice(hashes [][]byte) [][sha256.Size]byte {
+	var ret [][sha256.Size]byte
+	for _, hash := range hashes {
+		var h [sha256.Size]byte
+		copy(h[:], hash)
+		ret = append(ret, h)
+	}
+	return ret
+}
diff --git a/internal/options/options.go b/internal/options/options.go
deleted file mode 100644
index 3e253c5..0000000
--- a/internal/options/options.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package options
-
-import (
-	"encoding/json"
-	"flag"
-	"fmt"
-	"os"
-
-	"rgdd.se/silent-ct/internal/manager"
-	"rgdd.se/silent-ct/pkg/monitor"
-	"rgdd.se/silent-ct/pkg/server"
-)
-
-const usage = `Usage:
-
-  silent-ct [Options]
-
-Options:
-
-  -h, --help:      Output usage message and exit
-  -c, --config:    Path to a configuration file (Default: %s)
-  -l, --listen:    Listen address to receive submission on (Default: %s)
-  -s, --state:     Path to a directory where state is stored (Default: %s)
-
-Example configuration file:
-
-  {
-    "monitor": [
-      {
-        "wildcard": "example.org",
-        "excludes": [
-          "test"
-        ]
-      }
-    ],
-    "nodes": [
-      {
-        "name": "node_a",
-        "secret": "aaaa",
-        "issues": [
-          "example.org",
-          "www.example.org"
-        ]
-      }
-    ]
-  }
-
-`
-
-// Options are command-line options the user can specify
-type Options struct {
-	ListenAddr string
-	ConfigFile string
-	StateDir   string
-}
-
-func New(cmd string, args []string) (opts Options, err error) {
-	fs := flag.NewFlagSet(cmd, flag.ContinueOnError)
-	fs.Usage = func() {
-		fmt.Fprintf(os.Stderr, usage, server.DefaultConfigFile, server.DefaultAddress, manager.DefaultStateDir)
-	}
-	stringOpt(fs, &opts.ConfigFile, "config", "c", server.DefaultConfigFile)
-	stringOpt(fs, &opts.ListenAddr, "listen", "l", server.DefaultAddress)
-	stringOpt(fs, &opts.StateDir, "state", "s", manager.DefaultStateDir)
-	if err = fs.Parse(args); err != nil {
-		return opts, err
-	}
-
-	if opts.ConfigFile == "" {
-		return opts, fmt.Errorf("-c, --config: must not be an empty string")
-	}
-	if opts.StateDir == "" {
-		return opts, fmt.Errorf("-s, --state: must not be an empty string")
-	}
-	if opts.ListenAddr == "" {
-		return opts, fmt.Errorf("-l, --listen: must not be an empty string")
-	}
-	return opts, err
-}
-
-func stringOpt(fs *flag.FlagSet, opt *string, short, long, value string) {
-	fs.StringVar(opt, short, value, "")
-	fs.StringVar(opt, long, value, "")
-}
-
-type Config struct {
-	Monitor monitor.MatchWildcards `json:"monitor"`
-	Nodes   server.Nodes           `json:"nodes"`
-}
-
-func (c *Config) FromFile(fileName string) error {
-	b, err := os.ReadFile(fileName)
-	if err != nil {
-		return err
-	}
-	return json.Unmarshal(b, c)
-}
diff --git a/internal/x509util/x509util.go b/internal/x509util/x509util.go
deleted file mode 100644
index 912d1b4..0000000
--- a/internal/x509util/x509util.go
+++ /dev/null
@@ -1,44 +0,0 @@
-package x509util
-
-import (
-	"crypto/x509"
-	"encoding/pem"
-	"fmt"
-)
-
-// ParseChain parses a certificate chain in PEM format.  At least one
-// certificate must be in the chain.  The first certificate must be a leaf,
-// whereas all other certificates must CA certificates (intermdiates/roots).
-//
-// Note: it is not checked if the certificate chain's root is trusted or not.
-func ParseChain(b []byte) ([]x509.Certificate, error) {
-	var chain []x509.Certificate
-
-	for {
-		block, rest := pem.Decode(b)
-		if block == nil {
-			break
-		}
-		crt, err := x509.ParseCertificate(block.Bytes)
-		if err != nil {
-			return nil, fmt.Errorf("parse certificate: %v", err)
-		}
-
-		chain = append(chain, *crt)
-		b = rest
-	}
-
-	if len(chain) == 0 {
-		return nil, fmt.Errorf("no certificates in the provided chain")
-	}
-	if chain[0].IsCA {
-		return nil, fmt.Errorf("leaf certificate has the CA bit set")
-	}
-	for _, crt := range chain[1:] {
-		if !crt.IsCA {
-			return nil, fmt.Errorf("non-leaf certificate without the CA bit set")
-		}
-	}
-
-	return chain, nil
-}