2 files changed, 207 insertions, 0 deletions
diff --git a/pkg/storage/loglist/loglist.go b/pkg/storage/loglist/loglist.go
new file mode 100644
index 0000000..ccc63b0
--- /dev/null
+++ b/pkg/storage/loglist/loglist.go
@@ -0,0 +1,145 @@
+// Package loglist manages a list of logs to monitor.  The list of logs is based
+// on Google's signed list.  Logs can also be added and removed manually.
+package loglist
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"gitlab.torproject.org/rgdd/ct/pkg/metadata"
+	"rgdd.se/silent-ct/internal/ioutil"
+)
+
+type Config struct {
+	PermitBootstrap  bool   // Get and create initial log metadata if nothing valid is available on disk
+	MetadataFile     string // Path to a dynamically updated metadata file that can be read/written
+	HistoryDirectory string // Existing directory to store the history of downloaded metadata
+
+	// Optional
+	MetadataInFuture  time.Duration     // How wrong the metadata timestamp is allowed to be wrt. future dating
+	MetadataGetsStale time.Duration     // How long until the metadata is considered stale
+	MetadataIsRecent  time.Duration     // How long the metadata is considered recent
+	HTTPTimeout       time.Duration     // Timeout when fetching metadata
+	StaticLogs        []metadata.Log    // Takes precedence over the dynamically downloaded metadata
+	RemoveLogs        []metadata.LogKey // Logs in the downloaded metadata with these keys are ignored
+}
+
+type LogList struct {
+	cfg    Config
+	md     metadata.Metadata
+	source metadata.Loader
+}
+
+func New(cfg Config) (LogList, error) {
+	if cfg.MetadataInFuture == 0 {
+		cfg.MetadataInFuture = 24 * time.Hour
+	}
+	if cfg.MetadataGetsStale == 0 {
+		cfg.MetadataGetsStale = 30 * 24 * time.Hour
+	}
+	if cfg.MetadataIsRecent == 0 {
+		cfg.MetadataIsRecent = 1 * time.Hour
+	}
+	if cfg.HTTPTimeout == 0 {
+		cfg.HTTPTimeout = 10 * time.Second
+	}
+
+	for i, log := range cfg.StaticLogs {
+		if err := checkLog(log); err != nil {
+			return LogList{}, fmt.Errorf("static logs: index %d: %v", i, err)
+		}
+	}
+	for i, key := range cfg.RemoveLogs {
+		if _, err := key.ID(); err != nil {
+			return LogList{}, fmt.Errorf("remove logs: index %d: %v", i, err)
+		}
+	}
+
+	s := metadata.NewHTTPSource(metadata.HTTPSourceOptions{Name: "google"})
+	ll := LogList{cfg: cfg, source: &s}
+	if err := ioutil.DirectoriesExist([]string{cfg.HistoryDirectory}); err != nil {
+		return LogList{}, err
+	}
+	if err := ioutil.ReadJSON(cfg.MetadataFile, &ll.md); err != nil {
+		if !ll.cfg.PermitBootstrap {
+			return LogList{}, err
+		}
+		if _, _, err := ll.Update(context.Background()); err != nil {
+			return LogList{}, err
+		}
+	}
+	return ll, nil
+}
+
+func (ll *LogList) IsRecent() bool {
+	return time.Now().Before(ll.md.CreatedAt.Add(ll.cfg.MetadataIsRecent))
+}
+
+func (ll *LogList) IsStale() bool {
+	return time.Now().After(ll.md.CreatedAt.Add(ll.cfg.MetadataGetsStale))
+}
+
+func (ll *LogList) Generate() []metadata.Log {
+	var configure []metadata.Log
+
+	for _, log := range ll.cfg.StaticLogs {
+		configure = append(configure, log)
+	}
+	for _, operator := range ll.md.Operators {
+		for _, log := range operator.Logs {
+			if findKey(ll.cfg.RemoveLogs, log) {
+				continue // static configuration says to remove this log
+			}
+			if findLog(configure, log) {
+				continue // static configuration takes precedence
+			}
+			if skipLog(log) {
+				continue // not in a state where it makes sense to monitor
+			}
+			configure = append(configure, log)
+		}
+	}
+
+	return configure
+}
+
+func (ll *LogList) Update(ctx context.Context) (added []metadata.Log, removed []metadata.Log, err error) {
+	b, md, err := ll.archiveDynamic(ctx)
+	if err != nil {
+		return
+	}
+	if err = ioutil.CommitData(ll.cfg.MetadataFile, b); err != nil {
+		return
+	}
+
+	added, removed = metadataLogDiff(ll.md, md)
+	ll.md = md
+	return
+}
+
+func (ll *LogList) archiveDynamic(ctx context.Context) ([]byte, metadata.Metadata, error) {
+	llctx, cancel := context.WithTimeout(ctx, ll.cfg.HTTPTimeout)
+	defer cancel()
+
+	msg, sig, md, err := ll.source.Load(llctx)
+	if err != nil {
+		return nil, metadata.Metadata{}, err
+	}
+	if future := time.Now().Add(ll.cfg.MetadataInFuture); md.CreatedAt.After(future) {
+		return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is in the future", md.CreatedAt)
+	}
+	if md.CreatedAt.Before(ll.md.CreatedAt) {
+		return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is older than the current list", md.CreatedAt)
+	}
+
+	// FIXME: consider only archiving on major version bumps
+	path := fmt.Sprintf("%s/%s.json", ll.cfg.HistoryDirectory, time.Now().Format("2006-01-02_1504"))
+	if err := ioutil.CommitData(path, msg); err != nil {
+		return nil, metadata.Metadata{}, err
+	}
+	if err := ioutil.CommitData(path+".sig", sig); err != nil {
+		return nil, metadata.Metadata{}, err
+	}
+	return msg, md, nil
+}
diff --git a/pkg/storage/loglist/metadata.go b/pkg/storage/loglist/metadata.go
new file mode 100644
index 0000000..adacf81
--- /dev/null
+++ b/pkg/storage/loglist/metadata.go
@@ -0,0 +1,62 @@
+package loglist
+
+import "gitlab.torproject.org/rgdd/ct/pkg/metadata"
+
+// FIXME: helpers that should probably be in the upstream package
+
+func metadataFindLog(md metadata.Metadata, target metadata.Log) bool {
+	for _, operator := range md.Operators {
+		if findLog(operator.Logs, target) {
+			return true
+		}
+	}
+	return false
+}
+
+func findLog(logs []metadata.Log, target metadata.Log) bool {
+	targetID, _ := target.Key.ID()
+	for _, log := range logs {
+		id, _ := log.Key.ID()
+		if id == targetID {
+			return true
+		}
+	}
+	return false
+}
+
+func findKey(keys []metadata.LogKey, target metadata.Log) bool {
+	targetID, _ := target.Key.ID()
+	for _, key := range keys {
+		id, _ := key.ID()
+		if id == targetID {
+			return true
+		}
+	}
+	return false
+}
+
+func metadataLogDiff(initial, other metadata.Metadata) (added []metadata.Log, removed []metadata.Log) {
+	return metadataNewLogsIn(initial, other), metadataNewLogsIn(other, initial)
+}
+
+func metadataNewLogsIn(initial, other metadata.Metadata) (added []metadata.Log) {
+	for _, operator := range other.Operators {
+		for _, log := range operator.Logs {
+			if !metadataFindLog(initial, log) {
+				added = append(added, log)
+			}
+		}
+	}
+	return
+}
+
+func checkLog(log metadata.Log) error {
+	return nil // FIXME: check valid key, url, mmd, state
+}
+
+func skipLog(log metadata.Log) bool {
+	return log.State == nil || // logs without a state are considered misconfigured
+		log.State.Name == metadata.LogStatePending || // log is not yet relevant
+		log.State.Name == metadata.LogStateRetired || // log is not expected to be reachable
+		log.State.Name == metadata.LogStateRejected // log is not expected to be reachable
+}