Add drafty prototype

6 files changed, 583 insertions, 0 deletions

diff --git a/pkg/storage/errors.go b/pkg/storage/errors.go
new file mode 100644
index 0000000..3d7997f
--- /dev/null
+++ b/pkg/storage/errors.go
@@ -0,0 +1,5 @@
+package storage
+
+import "errors"
+
+var ErrorMonitorStateExists = errors.New("monitor state already exists on disk")
diff --git a/pkg/storage/index/index.go b/pkg/storage/index/index.go
new file mode 100644
index 0000000..ef9ad60
--- /dev/null
+++ b/pkg/storage/index/index.go
@@ -0,0 +1,103 @@
+// Package index provides an index of locally stored certificates.  If a method
+// succeeds, the index and the data that it tracks has been persisted to disk.
+// If a method does not succeed, restore from the persisted index on disk.
+package index
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"time"
+
+	"rgdd.se/silent-ct/internal/ioutil"
+	"rgdd.se/silent-ct/internal/monitor"
+	"rgdd.se/silent-ct/pkg/crtutil"
+)
+
+type Config struct {
+	PermitBootstrap bool   // Create a new index if a valid one does not exist on disk yet
+	IndexFile       string // Path to an index file that can be read/written
+	TrustDirectory  string // Absolute path to an existing directory where legitimate certificates are stored
+	MatchDirectory  string // Absolute path to an existing directory where matching certificates are stored
+
+	// Optional
+	AlertDelay time.Duration // Time before alerting on certificates that are unaccounted for
+}
+
+type Index struct {
+	mem index
+	cfg Config
+}
+
+func New(cfg Config) (Index, error) {
+	ix := Index{cfg: cfg}
+	if err := ioutil.DirectoriesExist([]string{cfg.TrustDirectory, cfg.MatchDirectory}); err != nil {
+		return Index{}, err
+	}
+	if err := ioutil.ReadJSON(cfg.IndexFile, &ix.mem); err != nil {
+		if !cfg.PermitBootstrap {
+			return Index{}, err
+		}
+
+		ix.mem = newIndex()
+		if err := ioutil.CommitJSON(cfg.IndexFile, ix.mem); err != nil {
+			return Index{}, err
+		}
+	}
+	return ix, ix.Validate()
+}
+
+func (ix *Index) AddChain(node string, pem []byte) error {
+	chain, err := crtutil.CertificateChainFromPEM(pem)
+	if err != nil {
+		return err
+	}
+
+	var crtID CertificateID
+	crtID.Set(chain[0])
+	path := fmt.Sprintf("%s/%s-%s.pem", ix.cfg.TrustDirectory, node, crtID)
+	if !ix.mem.addChain(crtID, path) {
+		return nil // duplicate
+	}
+
+	if ioutil.CommitData(path, pem); err != nil {
+		return err
+	}
+	return ioutil.CommitJSON(ix.cfg.IndexFile, ix.mem)
+}
+
+func (ix *Index) AddEntries(logID [sha256.Size]byte, entries []monitor.LogEntry) error {
+	addEntry := func(entry monitor.LogEntry) error {
+		crt, err := crtutil.CertificateFromLogEntry(entry.LeafData, entry.ExtraData)
+		if err != nil {
+			return err
+		}
+
+		var crtID CertificateID
+		crtID.Set(crt)
+		path := fmt.Sprintf("%s/%x-%d.json", ix.cfg.MatchDirectory, logID[:], entry.LeafIndex)
+		if !ix.mem.addEntry(crtID, path) {
+			return nil // duplicate
+		}
+
+		return ioutil.CommitJSON(path, entry)
+	}
+
+	for _, entry := range entries {
+		if err := addEntry(entry); err != nil {
+			return err
+		}
+	}
+	return ioutil.CommitJSON(ix.cfg.IndexFile, ix.mem)
+}
+
+func (ix *Index) TriggerAlerts() ([]CertificateInfo, error) {
+	alerts := ix.mem.triggerAlerts(ix.cfg.AlertDelay)
+	if len(alerts) == 0 {
+		return []CertificateInfo{}, nil
+	}
+	return alerts, ioutil.CommitJSON(ix.cfg.IndexFile, ix.mem)
+}
+
+func (index *Index) Validate() error {
+	return nil // FIXME: check that the index is populated with valid values
+}
diff --git a/pkg/storage/index/inmem.go b/pkg/storage/index/inmem.go
new file mode 100644
index 0000000..0a084bf
--- /dev/null
+++ b/pkg/storage/index/inmem.go
@@ -0,0 +1,113 @@
+package index
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/google/certificate-transparency-go/x509"
+	"rgdd.se/silent-ct/pkg/crtutil"
+)
+
+type CertificateID string
+
+func (crtID *CertificateID) Set(crt x509.Certificate) {
+	*crtID = CertificateID(crtutil.UniqueID(crt))
+}
+
+type CertificateInfo struct {
+	ObservedAt time.Time `json:"observed_at"`
+	StoredAt   string    `json:"stored_at"`
+}
+
+// index is an in-memory index of certificates
+type index struct {
+	Alerting   map[CertificateID][]CertificateInfo `json:"alerting"`   // Certificates that were not marked as "good" on time
+	Legitimate map[CertificateID][]CertificateInfo `json:"legitimate"` // Certificates that are considered "good"
+	Pending    map[CertificateID][]CertificateInfo `json:"pending"`    // Certificates that have yet to be marked as "good"
+}
+
+func newIndex() index {
+	return index{
+		Alerting:   make(map[CertificateID][]CertificateInfo),
+		Legitimate: make(map[CertificateID][]CertificateInfo),
+		Pending:    make(map[CertificateID][]CertificateInfo),
+	}
+}
+
+func (ix *index) JSONUnmarshal(b []byte) error {
+	type internal index
+	if err := json.Unmarshal(b, (*internal)(ix)); err != nil {
+		return err
+	}
+	for i, m := range []map[CertificateID][]CertificateInfo{ix.Alerting, ix.Legitimate, ix.Pending} {
+		if m == nil {
+			return fmt.Errorf("dictionary named %q is not in the index", []string{"alerting", "legitimate", "pending"}[i])
+		}
+	}
+	return nil
+}
+
+func (ix *index) triggerAlerts(delay time.Duration) []CertificateInfo {
+	var alerts []CertificateInfo
+
+	for key, certInfos := range ix.Pending {
+		certInfo := certInfos[0]
+		if time.Since(certInfo.ObservedAt) < delay {
+			continue
+		}
+
+		alerts = append(alerts, certInfo)
+		ix.Alerting[key] = certInfos
+		delete(ix.Pending, key)
+	}
+
+	return alerts
+}
+
+func (ix *index) addChain(crtID CertificateID, path string) bool {
+	if _, ok := ix.Legitimate[crtID]; ok {
+		return false // we already marked this certificate as "good"
+	}
+
+	entry := CertificateInfo{ObservedAt: time.Now(), StoredAt: path}
+	crtInfos := []CertificateInfo{entry}
+	if v, ok := ix.Alerting[crtID]; ok {
+		crtInfos = append(crtInfos, v...)
+		delete(ix.Alerting, crtID) // no longer alerting
+	} else if v, ok := ix.Pending[crtID]; ok {
+		crtInfos = append(crtInfos, v...)
+		delete(ix.Pending, crtID) // no longer pending
+	}
+
+	ix.Legitimate[crtID] = crtInfos
+	return true // index updated such that this certificate is marked as "good"
+}
+
+func (ix *index) addEntry(crtID CertificateID, path string) bool {
+	crtInfo := CertificateInfo{ObservedAt: time.Now(), StoredAt: path}
+	if _, ok := ix.Legitimate[crtID]; ok {
+		return add(ix.Legitimate, crtID, crtInfo)
+	} else if _, ok := ix.Alerting[crtID]; ok {
+		return add(ix.Alerting, crtID, crtInfo)
+	}
+	return add(ix.Pending, crtID, crtInfo)
+}
+
+func add(m map[CertificateID][]CertificateInfo, key CertificateID, value CertificateInfo) bool {
+	crtInfos, ok := m[key]
+	if !ok {
+		m[key] = []CertificateInfo{value}
+		return true
+	}
+
+	for _, crtInfo := range crtInfos {
+		if value.StoredAt == crtInfo.StoredAt {
+			return false // duplicate
+		}
+	}
+
+	crtInfos = append(crtInfos, value)
+	m[key] = crtInfos
+	return true
+}
diff --git a/pkg/storage/loglist/loglist.go b/pkg/storage/loglist/loglist.go
new file mode 100644
index 0000000..ccc63b0
--- /dev/null
+++ b/pkg/storage/loglist/loglist.go
@@ -0,0 +1,145 @@
+// Package loglist manages a list of logs to monitor.  The list of logs is based
+// on Google's signed list.  Logs can also be added and removed manually.
+package loglist
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"gitlab.torproject.org/rgdd/ct/pkg/metadata"
+	"rgdd.se/silent-ct/internal/ioutil"
+)
+
+type Config struct {
+	PermitBootstrap  bool   // Get and create initial log metadata if nothing valid is available on disk
+	MetadataFile     string // Path to a dynamically updated metadata file that can be read/written
+	HistoryDirectory string // Existing directory to store the history of downloaded metadata
+
+	// Optional
+	MetadataInFuture  time.Duration     // How wrong the metadata timestamp is allowed to be wrt. future dating
+	MetadataGetsStale time.Duration     // How long until the metadata is considered stale
+	MetadataIsRecent  time.Duration     // How long the metadata is considered recent
+	HTTPTimeout       time.Duration     // Timeout when fetching metadata
+	StaticLogs        []metadata.Log    // Takes precedence over the dynamically downloaded metadata
+	RemoveLogs        []metadata.LogKey // Logs in the downloaded metadata with these keys are ignored
+}
+
+type LogList struct {
+	cfg    Config
+	md     metadata.Metadata
+	source metadata.Loader
+}
+
+func New(cfg Config) (LogList, error) {
+	if cfg.MetadataInFuture == 0 {
+		cfg.MetadataInFuture = 24 * time.Hour
+	}
+	if cfg.MetadataGetsStale == 0 {
+		cfg.MetadataGetsStale = 30 * 24 * time.Hour
+	}
+	if cfg.MetadataIsRecent == 0 {
+		cfg.MetadataIsRecent = 1 * time.Hour
+	}
+	if cfg.HTTPTimeout == 0 {
+		cfg.HTTPTimeout = 10 * time.Second
+	}
+
+	for i, log := range cfg.StaticLogs {
+		if err := checkLog(log); err != nil {
+			return LogList{}, fmt.Errorf("static logs: index %d: %v", i, err)
+		}
+	}
+	for i, key := range cfg.RemoveLogs {
+		if _, err := key.ID(); err != nil {
+			return LogList{}, fmt.Errorf("remove logs: index %d: %v", i, err)
+		}
+	}
+
+	s := metadata.NewHTTPSource(metadata.HTTPSourceOptions{Name: "google"})
+	ll := LogList{cfg: cfg, source: &s}
+	if err := ioutil.DirectoriesExist([]string{cfg.HistoryDirectory}); err != nil {
+		return LogList{}, err
+	}
+	if err := ioutil.ReadJSON(cfg.MetadataFile, &ll.md); err != nil {
+		if !ll.cfg.PermitBootstrap {
+			return LogList{}, err
+		}
+		if _, _, err := ll.Update(context.Background()); err != nil {
+			return LogList{}, err
+		}
+	}
+	return ll, nil
+}
+
+func (ll *LogList) IsRecent() bool {
+	return time.Now().Before(ll.md.CreatedAt.Add(ll.cfg.MetadataIsRecent))
+}
+
+func (ll *LogList) IsStale() bool {
+	return time.Now().After(ll.md.CreatedAt.Add(ll.cfg.MetadataGetsStale))
+}
+
+func (ll *LogList) Generate() []metadata.Log {
+	var configure []metadata.Log
+
+	for _, log := range ll.cfg.StaticLogs {
+		configure = append(configure, log)
+	}
+	for _, operator := range ll.md.Operators {
+		for _, log := range operator.Logs {
+			if findKey(ll.cfg.RemoveLogs, log) {
+				continue // static configuration says to remove this log
+			}
+			if findLog(configure, log) {
+				continue // static configuration takes precedence
+			}
+			if skipLog(log) {
+				continue // not in a state where it makes sense to monitor
+			}
+			configure = append(configure, log)
+		}
+	}
+
+	return configure
+}
+
+func (ll *LogList) Update(ctx context.Context) (added []metadata.Log, removed []metadata.Log, err error) {
+	b, md, err := ll.archiveDynamic(ctx)
+	if err != nil {
+		return
+	}
+	if err = ioutil.CommitData(ll.cfg.MetadataFile, b); err != nil {
+		return
+	}
+
+	added, removed = metadataLogDiff(ll.md, md)
+	ll.md = md
+	return
+}
+
+func (ll *LogList) archiveDynamic(ctx context.Context) ([]byte, metadata.Metadata, error) {
+	llctx, cancel := context.WithTimeout(ctx, ll.cfg.HTTPTimeout)
+	defer cancel()
+
+	msg, sig, md, err := ll.source.Load(llctx)
+	if err != nil {
+		return nil, metadata.Metadata{}, err
+	}
+	if future := time.Now().Add(ll.cfg.MetadataInFuture); md.CreatedAt.After(future) {
+		return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is in the future", md.CreatedAt)
+	}
+	if md.CreatedAt.Before(ll.md.CreatedAt) {
+		return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is older than the current list", md.CreatedAt)
+	}
+
+	// FIXME: consider only archiving on major version bumps
+	path := fmt.Sprintf("%s/%s.json", ll.cfg.HistoryDirectory, time.Now().Format("2006-01-02_1504"))
+	if err := ioutil.CommitData(path, msg); err != nil {
+		return nil, metadata.Metadata{}, err
+	}
+	if err := ioutil.CommitData(path+".sig", sig); err != nil {
+		return nil, metadata.Metadata{}, err
+	}
+	return msg, md, nil
+}
diff --git a/pkg/storage/loglist/metadata.go b/pkg/storage/loglist/metadata.go
new file mode 100644
index 0000000..adacf81
--- /dev/null
+++ b/pkg/storage/loglist/metadata.go
@@ -0,0 +1,62 @@
+package loglist
+
+import "gitlab.torproject.org/rgdd/ct/pkg/metadata"
+
+// FIXME: helpers that should probably be in the upstream package
+
+func metadataFindLog(md metadata.Metadata, target metadata.Log) bool {
+	for _, operator := range md.Operators {
+		if findLog(operator.Logs, target) {
+			return true
+		}
+	}
+	return false
+}
+
+func findLog(logs []metadata.Log, target metadata.Log) bool {
+	targetID, _ := target.Key.ID()
+	for _, log := range logs {
+		id, _ := log.Key.ID()
+		if id == targetID {
+			return true
+		}
+	}
+	return false
+}
+
+func findKey(keys []metadata.LogKey, target metadata.Log) bool {
+	targetID, _ := target.Key.ID()
+	for _, key := range keys {
+		id, _ := key.ID()
+		if id == targetID {
+			return true
+		}
+	}
+	return false
+}
+
+func metadataLogDiff(initial, other metadata.Metadata) (added []metadata.Log, removed []metadata.Log) {
+	return metadataNewLogsIn(initial, other), metadataNewLogsIn(other, initial)
+}
+
+func metadataNewLogsIn(initial, other metadata.Metadata) (added []metadata.Log) {
+	for _, operator := range other.Operators {
+		for _, log := range operator.Logs {
+			if !metadataFindLog(initial, log) {
+				added = append(added, log)
+			}
+		}
+	}
+	return
+}
+
+func checkLog(log metadata.Log) error {
+	return nil // FIXME: check valid key, url, mmd, state
+}
+
+func skipLog(log metadata.Log) bool {
+	return log.State == nil || // logs without a state are considered misconfigured
+		log.State.Name == metadata.LogStatePending || // log is not yet relevant
+		log.State.Name == metadata.LogStateRetired || // log is not expected to be reachable
+		log.State.Name == metadata.LogStateRejected // log is not expected to be reachable
+}
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
new file mode 100644
index 0000000..5e28aca
--- /dev/null
+++ b/pkg/storage/storage.go
@@ -0,0 +1,155 @@
+// Package storage manages an index of certificates, a dynamically updated log
+// list, and a monitor's state on the local file system in a single directory.
+package storage
+
+import (
+	"context"
+	"crypto/sha256"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/google/certificate-transparency-go/client"
+	"github.com/google/certificate-transparency-go/jsonclient"
+	"gitlab.torproject.org/rgdd/ct/pkg/metadata"
+	"rgdd.se/silent-ct/internal/ioutil"
+	"rgdd.se/silent-ct/internal/monitor"
+	"rgdd.se/silent-ct/pkg/storage/index"
+	"rgdd.se/silent-ct/pkg/storage/loglist"
+)
+
+type Config struct {
+	Bootstrap bool   // Whether a new storage should be bootstrapped in a non-existing directory
+	Directory string // Path to a directory where everything will be stored
+
+	// Optional
+	AlertDelay  time.Duration     // Time before alerting on certificates that are unaccounted for
+	StaticLogs  []metadata.Log    // Static logs to configure in loglist
+	RemoveLogs  []metadata.LogKey // Keys of logs to omit in loglist
+	HTTPTimeout time.Duration     // HTTP timeout used when bootstrapping logs
+}
+
+func (cfg *Config) CertificateIndexFile() string           { return cfg.Directory + "/crt_index.json" }
+func (cfg *Config) LegitimateCertificateDirectory() string { return cfg.Directory + "/crt_trusted" }
+func (cfg *Config) DiscoveredCertificateDirectory() string { return cfg.Directory + "/crt_found" }
+func (cfg *Config) MetadataFile() string                   { return cfg.Directory + "/metadata.json" }
+func (cfg *Config) MetadataHistoryDirectory() string       { return cfg.Directory + "/metadata_history" }
+func (cfg *Config) MonitorStateDirectory() string          { return cfg.Directory + "/monitor_state" }
+func (cfg *Config) MonitorStateFile(logID [sha256.Size]byte) string {
+	return fmt.Sprintf("%s/%x.json", cfg.MonitorStateDirectory(), logID[:])
+}
+
+func (cfg *Config) directories() []string {
+	return []string{
+		cfg.Directory,
+		cfg.LegitimateCertificateDirectory(),
+		cfg.DiscoveredCertificateDirectory(),
+		cfg.MetadataHistoryDirectory(),
+		cfg.MonitorStateDirectory(),
+	}
+}
+
+func (cfg *Config) configure() error {
+	if cfg.Directory == "" {
+		return fmt.Errorf("directory is required")
+	}
+	if cfg.HTTPTimeout == 0 {
+		cfg.HTTPTimeout = 10 * time.Second
+	}
+
+	path, err := filepath.Abs(cfg.Directory)
+	if err != nil {
+		return err
+	}
+	cfg.Directory = path
+	if err := ioutil.DirectoriesExist(cfg.directories()); err != nil {
+		if !cfg.Bootstrap {
+			return err
+		}
+		return ioutil.CreateDirectories(cfg.directories())
+	}
+	return nil
+}
+
+type Storage struct {
+	Config
+	index.Index
+	loglist.LogList
+}
+
+func New(cfg Config) (Storage, error) {
+	err := cfg.configure()
+	if err != nil {
+		return Storage{}, err
+	}
+
+	s := Storage{Config: cfg}
+	if s.Index, err = index.New(index.Config{
+		PermitBootstrap: cfg.Bootstrap,
+		IndexFile:       cfg.CertificateIndexFile(),
+		TrustDirectory:  cfg.LegitimateCertificateDirectory(),
+		MatchDirectory:  cfg.DiscoveredCertificateDirectory(),
+		AlertDelay:      cfg.AlertDelay,
+	}); err != nil {
+		return Storage{}, err
+	}
+
+	if s.LogList, err = loglist.New(loglist.Config{
+		PermitBootstrap:  cfg.Bootstrap,
+		MetadataFile:     cfg.MetadataFile(),
+		HistoryDirectory: cfg.MetadataHistoryDirectory(),
+		StaticLogs:       cfg.StaticLogs,
+		RemoveLogs:       cfg.RemoveLogs,
+	}); err != nil {
+		return Storage{}, err
+	}
+
+	return s, err
+}
+
+func (s *Storage) BootstrapLog(ctx context.Context, log metadata.Log, skipBacklog bool) (monitor.State, error) {
+	storedState, err := s.GetMonitorState(log)
+	if err == nil {
+		return storedState, ErrorMonitorStateExists
+	}
+
+	key, err := x509.MarshalPKIXPublicKey(log.Key.Public)
+	if err != nil {
+		return monitor.State{}, err
+	}
+	cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{PublicKeyDER: key})
+	if err != nil {
+		return monitor.State{}, err
+	}
+
+	sctx, cancel := context.WithTimeout(ctx, s.Config.HTTPTimeout)
+	defer cancel()
+	sth, err := cli.GetSTH(sctx)
+	if err != nil {
+		return monitor.State{}, err
+	}
+	id, _ := log.Key.ID()
+	sth.LogID = id
+
+	state := monitor.State{SignedTreeHead: *sth}
+	if skipBacklog {
+		state.NextIndex = sth.TreeSize
+	}
+	return state, s.SetMonitorState(id, state)
+}
+
+func (s *Storage) SetMonitorState(logID [sha256.Size]byte, state monitor.State) error {
+	return ioutil.CommitJSON(s.MonitorStateFile(logID), state)
+}
+
+func (s *Storage) GetMonitorState(log metadata.Log) (monitor.State, error) {
+	id, err := log.Key.ID()
+	if err != nil {
+		return monitor.State{}, err
+	}
+
+	state := monitor.State{}
+	return state, ioutil.ReadJSON(s.MonitorStateFile(id), &state)
+}