From e18d36ebae30536c77c61cd5da123991e0ca1629 Mon Sep 17 00:00:00 2001 From: Rasmus Dahlberg Date: Sun, 31 Dec 2023 09:39:25 +0100 Subject: Add drafty prototype --- pkg/storage/errors.go | 5 ++ pkg/storage/index/index.go | 103 ++++++++++++++++++++++++++ pkg/storage/index/inmem.go | 113 +++++++++++++++++++++++++++++ pkg/storage/loglist/loglist.go | 145 +++++++++++++++++++++++++++++++++++++ pkg/storage/loglist/metadata.go | 62 ++++++++++++++++ pkg/storage/storage.go | 155 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 583 insertions(+) create mode 100644 pkg/storage/errors.go create mode 100644 pkg/storage/index/index.go create mode 100644 pkg/storage/index/inmem.go create mode 100644 pkg/storage/loglist/loglist.go create mode 100644 pkg/storage/loglist/metadata.go create mode 100644 pkg/storage/storage.go (limited to 'pkg/storage') diff --git a/pkg/storage/errors.go b/pkg/storage/errors.go new file mode 100644 index 0000000..3d7997f --- /dev/null +++ b/pkg/storage/errors.go @@ -0,0 +1,5 @@ +package storage + +import "errors" + +var ErrorMonitorStateExists = errors.New("monitor state already exists on disk") diff --git a/pkg/storage/index/index.go b/pkg/storage/index/index.go new file mode 100644 index 0000000..ef9ad60 --- /dev/null +++ b/pkg/storage/index/index.go @@ -0,0 +1,103 @@ +// Package index provides an index of locally stored certificates. If a method +// succeeds, the index and the data that it tracks has been persisted to disk. +// If a method does not succeed, restore from the persisted index on disk. +package index + +import ( + "crypto/sha256" + "fmt" + "time" + + "rgdd.se/silent-ct/internal/ioutil" + "rgdd.se/silent-ct/internal/monitor" + "rgdd.se/silent-ct/pkg/crtutil" +) + +type Config struct { + PermitBootstrap bool // Create a new index if a valid one does not exist on disk yet + IndexFile string // Path to an index file that can be read/written + TrustDirectory string // Absolute path to an existing directory where legitimate certificates are stored + MatchDirectory string // Absolute path to an existing directory where matching certificates are stored + + // Optional + AlertDelay time.Duration // Time before alerting on certificates that are unaccounted for +} + +type Index struct { + mem index + cfg Config +} + +func New(cfg Config) (Index, error) { + ix := Index{cfg: cfg} + if err := ioutil.DirectoriesExist([]string{cfg.TrustDirectory, cfg.MatchDirectory}); err != nil { + return Index{}, err + } + if err := ioutil.ReadJSON(cfg.IndexFile, &ix.mem); err != nil { + if !cfg.PermitBootstrap { + return Index{}, err + } + + ix.mem = newIndex() + if err := ioutil.CommitJSON(cfg.IndexFile, ix.mem); err != nil { + return Index{}, err + } + } + return ix, ix.Validate() +} + +func (ix *Index) AddChain(node string, pem []byte) error { + chain, err := crtutil.CertificateChainFromPEM(pem) + if err != nil { + return err + } + + var crtID CertificateID + crtID.Set(chain[0]) + path := fmt.Sprintf("%s/%s-%s.pem", ix.cfg.TrustDirectory, node, crtID) + if !ix.mem.addChain(crtID, path) { + return nil // duplicate + } + + if ioutil.CommitData(path, pem); err != nil { + return err + } + return ioutil.CommitJSON(ix.cfg.IndexFile, ix.mem) +} + +func (ix *Index) AddEntries(logID [sha256.Size]byte, entries []monitor.LogEntry) error { + addEntry := func(entry monitor.LogEntry) error { + crt, err := crtutil.CertificateFromLogEntry(entry.LeafData, entry.ExtraData) + if err != nil { + return err + } + + var crtID CertificateID + crtID.Set(crt) + path := fmt.Sprintf("%s/%x-%d.json", ix.cfg.MatchDirectory, logID[:], entry.LeafIndex) + if !ix.mem.addEntry(crtID, path) { + return nil // duplicate + } + + return ioutil.CommitJSON(path, entry) + } + + for _, entry := range entries { + if err := addEntry(entry); err != nil { + return err + } + } + return ioutil.CommitJSON(ix.cfg.IndexFile, ix.mem) +} + +func (ix *Index) TriggerAlerts() ([]CertificateInfo, error) { + alerts := ix.mem.triggerAlerts(ix.cfg.AlertDelay) + if len(alerts) == 0 { + return []CertificateInfo{}, nil + } + return alerts, ioutil.CommitJSON(ix.cfg.IndexFile, ix.mem) +} + +func (index *Index) Validate() error { + return nil // FIXME: check that the index is populated with valid values +} diff --git a/pkg/storage/index/inmem.go b/pkg/storage/index/inmem.go new file mode 100644 index 0000000..0a084bf --- /dev/null +++ b/pkg/storage/index/inmem.go @@ -0,0 +1,113 @@ +package index + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/google/certificate-transparency-go/x509" + "rgdd.se/silent-ct/pkg/crtutil" +) + +type CertificateID string + +func (crtID *CertificateID) Set(crt x509.Certificate) { + *crtID = CertificateID(crtutil.UniqueID(crt)) +} + +type CertificateInfo struct { + ObservedAt time.Time `json:"observed_at"` + StoredAt string `json:"stored_at"` +} + +// index is an in-memory index of certificates +type index struct { + Alerting map[CertificateID][]CertificateInfo `json:"alerting"` // Certificates that were not marked as "good" on time + Legitimate map[CertificateID][]CertificateInfo `json:"legitimate"` // Certificates that are considered "good" + Pending map[CertificateID][]CertificateInfo `json:"pending"` // Certificates that have yet to be marked as "good" +} + +func newIndex() index { + return index{ + Alerting: make(map[CertificateID][]CertificateInfo), + Legitimate: make(map[CertificateID][]CertificateInfo), + Pending: make(map[CertificateID][]CertificateInfo), + } +} + +func (ix *index) JSONUnmarshal(b []byte) error { + type internal index + if err := json.Unmarshal(b, (*internal)(ix)); err != nil { + return err + } + for i, m := range []map[CertificateID][]CertificateInfo{ix.Alerting, ix.Legitimate, ix.Pending} { + if m == nil { + return fmt.Errorf("dictionary named %q is not in the index", []string{"alerting", "legitimate", "pending"}[i]) + } + } + return nil +} + +func (ix *index) triggerAlerts(delay time.Duration) []CertificateInfo { + var alerts []CertificateInfo + + for key, certInfos := range ix.Pending { + certInfo := certInfos[0] + if time.Since(certInfo.ObservedAt) < delay { + continue + } + + alerts = append(alerts, certInfo) + ix.Alerting[key] = certInfos + delete(ix.Pending, key) + } + + return alerts +} + +func (ix *index) addChain(crtID CertificateID, path string) bool { + if _, ok := ix.Legitimate[crtID]; ok { + return false // we already marked this certificate as "good" + } + + entry := CertificateInfo{ObservedAt: time.Now(), StoredAt: path} + crtInfos := []CertificateInfo{entry} + if v, ok := ix.Alerting[crtID]; ok { + crtInfos = append(crtInfos, v...) + delete(ix.Alerting, crtID) // no longer alerting + } else if v, ok := ix.Pending[crtID]; ok { + crtInfos = append(crtInfos, v...) + delete(ix.Pending, crtID) // no longer pending + } + + ix.Legitimate[crtID] = crtInfos + return true // index updated such that this certificate is marked as "good" +} + +func (ix *index) addEntry(crtID CertificateID, path string) bool { + crtInfo := CertificateInfo{ObservedAt: time.Now(), StoredAt: path} + if _, ok := ix.Legitimate[crtID]; ok { + return add(ix.Legitimate, crtID, crtInfo) + } else if _, ok := ix.Alerting[crtID]; ok { + return add(ix.Alerting, crtID, crtInfo) + } + return add(ix.Pending, crtID, crtInfo) +} + +func add(m map[CertificateID][]CertificateInfo, key CertificateID, value CertificateInfo) bool { + crtInfos, ok := m[key] + if !ok { + m[key] = []CertificateInfo{value} + return true + } + + for _, crtInfo := range crtInfos { + if value.StoredAt == crtInfo.StoredAt { + return false // duplicate + } + } + + crtInfos = append(crtInfos, value) + m[key] = crtInfos + return true +} diff --git a/pkg/storage/loglist/loglist.go b/pkg/storage/loglist/loglist.go new file mode 100644 index 0000000..ccc63b0 --- /dev/null +++ b/pkg/storage/loglist/loglist.go @@ -0,0 +1,145 @@ +// Package loglist manages a list of logs to monitor. The list of logs is based +// on Google's signed list. Logs can also be added and removed manually. +package loglist + +import ( + "context" + "fmt" + "time" + + "gitlab.torproject.org/rgdd/ct/pkg/metadata" + "rgdd.se/silent-ct/internal/ioutil" +) + +type Config struct { + PermitBootstrap bool // Get and create initial log metadata if nothing valid is available on disk + MetadataFile string // Path to a dynamically updated metadata file that can be read/written + HistoryDirectory string // Existing directory to store the history of downloaded metadata + + // Optional + MetadataInFuture time.Duration // How wrong the metadata timestamp is allowed to be wrt. future dating + MetadataGetsStale time.Duration // How long until the metadata is considered stale + MetadataIsRecent time.Duration // How long the metadata is considered recent + HTTPTimeout time.Duration // Timeout when fetching metadata + StaticLogs []metadata.Log // Takes precedence over the dynamically downloaded metadata + RemoveLogs []metadata.LogKey // Logs in the downloaded metadata with these keys are ignored +} + +type LogList struct { + cfg Config + md metadata.Metadata + source metadata.Loader +} + +func New(cfg Config) (LogList, error) { + if cfg.MetadataInFuture == 0 { + cfg.MetadataInFuture = 24 * time.Hour + } + if cfg.MetadataGetsStale == 0 { + cfg.MetadataGetsStale = 30 * 24 * time.Hour + } + if cfg.MetadataIsRecent == 0 { + cfg.MetadataIsRecent = 1 * time.Hour + } + if cfg.HTTPTimeout == 0 { + cfg.HTTPTimeout = 10 * time.Second + } + + for i, log := range cfg.StaticLogs { + if err := checkLog(log); err != nil { + return LogList{}, fmt.Errorf("static logs: index %d: %v", i, err) + } + } + for i, key := range cfg.RemoveLogs { + if _, err := key.ID(); err != nil { + return LogList{}, fmt.Errorf("remove logs: index %d: %v", i, err) + } + } + + s := metadata.NewHTTPSource(metadata.HTTPSourceOptions{Name: "google"}) + ll := LogList{cfg: cfg, source: &s} + if err := ioutil.DirectoriesExist([]string{cfg.HistoryDirectory}); err != nil { + return LogList{}, err + } + if err := ioutil.ReadJSON(cfg.MetadataFile, &ll.md); err != nil { + if !ll.cfg.PermitBootstrap { + return LogList{}, err + } + if _, _, err := ll.Update(context.Background()); err != nil { + return LogList{}, err + } + } + return ll, nil +} + +func (ll *LogList) IsRecent() bool { + return time.Now().Before(ll.md.CreatedAt.Add(ll.cfg.MetadataIsRecent)) +} + +func (ll *LogList) IsStale() bool { + return time.Now().After(ll.md.CreatedAt.Add(ll.cfg.MetadataGetsStale)) +} + +func (ll *LogList) Generate() []metadata.Log { + var configure []metadata.Log + + for _, log := range ll.cfg.StaticLogs { + configure = append(configure, log) + } + for _, operator := range ll.md.Operators { + for _, log := range operator.Logs { + if findKey(ll.cfg.RemoveLogs, log) { + continue // static configuration says to remove this log + } + if findLog(configure, log) { + continue // static configuration takes precedence + } + if skipLog(log) { + continue // not in a state where it makes sense to monitor + } + configure = append(configure, log) + } + } + + return configure +} + +func (ll *LogList) Update(ctx context.Context) (added []metadata.Log, removed []metadata.Log, err error) { + b, md, err := ll.archiveDynamic(ctx) + if err != nil { + return + } + if err = ioutil.CommitData(ll.cfg.MetadataFile, b); err != nil { + return + } + + added, removed = metadataLogDiff(ll.md, md) + ll.md = md + return +} + +func (ll *LogList) archiveDynamic(ctx context.Context) ([]byte, metadata.Metadata, error) { + llctx, cancel := context.WithTimeout(ctx, ll.cfg.HTTPTimeout) + defer cancel() + + msg, sig, md, err := ll.source.Load(llctx) + if err != nil { + return nil, metadata.Metadata{}, err + } + if future := time.Now().Add(ll.cfg.MetadataInFuture); md.CreatedAt.After(future) { + return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is in the future", md.CreatedAt) + } + if md.CreatedAt.Before(ll.md.CreatedAt) { + return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is older than the current list", md.CreatedAt) + } + + // FIXME: consider only archiving on major version bumps + path := fmt.Sprintf("%s/%s.json", ll.cfg.HistoryDirectory, time.Now().Format("2006-01-02_1504")) + if err := ioutil.CommitData(path, msg); err != nil { + return nil, metadata.Metadata{}, err + } + if err := ioutil.CommitData(path+".sig", sig); err != nil { + return nil, metadata.Metadata{}, err + } + return msg, md, nil +} diff --git a/pkg/storage/loglist/metadata.go b/pkg/storage/loglist/metadata.go new file mode 100644 index 0000000..adacf81 --- /dev/null +++ b/pkg/storage/loglist/metadata.go @@ -0,0 +1,62 @@ +package loglist + +import "gitlab.torproject.org/rgdd/ct/pkg/metadata" + +// FIXME: helpers that should probably be in the upstream package + +func metadataFindLog(md metadata.Metadata, target metadata.Log) bool { + for _, operator := range md.Operators { + if findLog(operator.Logs, target) { + return true + } + } + return false +} + +func findLog(logs []metadata.Log, target metadata.Log) bool { + targetID, _ := target.Key.ID() + for _, log := range logs { + id, _ := log.Key.ID() + if id == targetID { + return true + } + } + return false +} + +func findKey(keys []metadata.LogKey, target metadata.Log) bool { + targetID, _ := target.Key.ID() + for _, key := range keys { + id, _ := key.ID() + if id == targetID { + return true + } + } + return false +} + +func metadataLogDiff(initial, other metadata.Metadata) (added []metadata.Log, removed []metadata.Log) { + return metadataNewLogsIn(initial, other), metadataNewLogsIn(other, initial) +} + +func metadataNewLogsIn(initial, other metadata.Metadata) (added []metadata.Log) { + for _, operator := range other.Operators { + for _, log := range operator.Logs { + if !metadataFindLog(initial, log) { + added = append(added, log) + } + } + } + return +} + +func checkLog(log metadata.Log) error { + return nil // FIXME: check valid key, url, mmd, state +} + +func skipLog(log metadata.Log) bool { + return log.State == nil || // logs without a state are considered misconfigured + log.State.Name == metadata.LogStatePending || // log is not yet relevant + log.State.Name == metadata.LogStateRetired || // log is not expected to be reachable + log.State.Name == metadata.LogStateRejected // log is not expected to be reachable +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go new file mode 100644 index 0000000..5e28aca --- /dev/null +++ b/pkg/storage/storage.go @@ -0,0 +1,155 @@ +// Package storage manages an index of certificates, a dynamically updated log +// list, and a monitor's state on the local file system in a single directory. +package storage + +import ( + "context" + "crypto/sha256" + "crypto/x509" + "fmt" + "net/http" + "path/filepath" + "time" + + "github.com/google/certificate-transparency-go/client" + "github.com/google/certificate-transparency-go/jsonclient" + "gitlab.torproject.org/rgdd/ct/pkg/metadata" + "rgdd.se/silent-ct/internal/ioutil" + "rgdd.se/silent-ct/internal/monitor" + "rgdd.se/silent-ct/pkg/storage/index" + "rgdd.se/silent-ct/pkg/storage/loglist" +) + +type Config struct { + Bootstrap bool // Whether a new storage should be bootstrapped in a non-existing directory + Directory string // Path to a directory where everything will be stored + + // Optional + AlertDelay time.Duration // Time before alerting on certificates that are unaccounted for + StaticLogs []metadata.Log // Static logs to configure in loglist + RemoveLogs []metadata.LogKey // Keys of logs to omit in loglist + HTTPTimeout time.Duration // HTTP timeout used when bootstrapping logs +} + +func (cfg *Config) CertificateIndexFile() string { return cfg.Directory + "/crt_index.json" } +func (cfg *Config) LegitimateCertificateDirectory() string { return cfg.Directory + "/crt_trusted" } +func (cfg *Config) DiscoveredCertificateDirectory() string { return cfg.Directory + "/crt_found" } +func (cfg *Config) MetadataFile() string { return cfg.Directory + "/metadata.json" } +func (cfg *Config) MetadataHistoryDirectory() string { return cfg.Directory + "/metadata_history" } +func (cfg *Config) MonitorStateDirectory() string { return cfg.Directory + "/monitor_state" } +func (cfg *Config) MonitorStateFile(logID [sha256.Size]byte) string { + return fmt.Sprintf("%s/%x.json", cfg.MonitorStateDirectory(), logID[:]) +} + +func (cfg *Config) directories() []string { + return []string{ + cfg.Directory, + cfg.LegitimateCertificateDirectory(), + cfg.DiscoveredCertificateDirectory(), + cfg.MetadataHistoryDirectory(), + cfg.MonitorStateDirectory(), + } +} + +func (cfg *Config) configure() error { + if cfg.Directory == "" { + return fmt.Errorf("directory is required") + } + if cfg.HTTPTimeout == 0 { + cfg.HTTPTimeout = 10 * time.Second + } + + path, err := filepath.Abs(cfg.Directory) + if err != nil { + return err + } + cfg.Directory = path + if err := ioutil.DirectoriesExist(cfg.directories()); err != nil { + if !cfg.Bootstrap { + return err + } + return ioutil.CreateDirectories(cfg.directories()) + } + return nil +} + +type Storage struct { + Config + index.Index + loglist.LogList +} + +func New(cfg Config) (Storage, error) { + err := cfg.configure() + if err != nil { + return Storage{}, err + } + + s := Storage{Config: cfg} + if s.Index, err = index.New(index.Config{ + PermitBootstrap: cfg.Bootstrap, + IndexFile: cfg.CertificateIndexFile(), + TrustDirectory: cfg.LegitimateCertificateDirectory(), + MatchDirectory: cfg.DiscoveredCertificateDirectory(), + AlertDelay: cfg.AlertDelay, + }); err != nil { + return Storage{}, err + } + + if s.LogList, err = loglist.New(loglist.Config{ + PermitBootstrap: cfg.Bootstrap, + MetadataFile: cfg.MetadataFile(), + HistoryDirectory: cfg.MetadataHistoryDirectory(), + StaticLogs: cfg.StaticLogs, + RemoveLogs: cfg.RemoveLogs, + }); err != nil { + return Storage{}, err + } + + return s, err +} + +func (s *Storage) BootstrapLog(ctx context.Context, log metadata.Log, skipBacklog bool) (monitor.State, error) { + storedState, err := s.GetMonitorState(log) + if err == nil { + return storedState, ErrorMonitorStateExists + } + + key, err := x509.MarshalPKIXPublicKey(log.Key.Public) + if err != nil { + return monitor.State{}, err + } + cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{PublicKeyDER: key}) + if err != nil { + return monitor.State{}, err + } + + sctx, cancel := context.WithTimeout(ctx, s.Config.HTTPTimeout) + defer cancel() + sth, err := cli.GetSTH(sctx) + if err != nil { + return monitor.State{}, err + } + id, _ := log.Key.ID() + sth.LogID = id + + state := monitor.State{SignedTreeHead: *sth} + if skipBacklog { + state.NextIndex = sth.TreeSize + } + return state, s.SetMonitorState(id, state) +} + +func (s *Storage) SetMonitorState(logID [sha256.Size]byte, state monitor.State) error { + return ioutil.CommitJSON(s.MonitorStateFile(logID), state) +} + +func (s *Storage) GetMonitorState(log metadata.Log) (monitor.State, error) { + id, err := log.Key.ID() + if err != nil { + return monitor.State{}, err + } + + state := monitor.State{} + return state, ioutil.ReadJSON(s.MonitorStateFile(id), &state) +} -- cgit v1.2.3