From e18d36ebae30536c77c61cd5da123991e0ca1629 Mon Sep 17 00:00:00 2001 From: Rasmus Dahlberg Date: Sun, 31 Dec 2023 09:39:25 +0100 Subject: Add drafty prototype --- internal/feedback/feedback.go | 127 ++++++++++++++++++++ internal/flagopt/flagopt.go | 26 ++++ internal/ioutil/ioutil.go | 56 +++++++++ internal/logger/logger.go | 96 +++++++++++++++ internal/manager/helpers.go | 52 -------- internal/manager/manager.go | 219 ++++++++++++++++++++++++++-------- internal/merkle/TODO | 1 - internal/merkle/compact.go | 115 ------------------ internal/merkle/merkle.go | 271 ------------------------------------------ internal/monitor/chunks.go | 89 ++++++++++++++ internal/monitor/matcher.go | 13 ++ internal/monitor/monitor.go | 173 +++++++++++++++++++++++++++ internal/monitor/tail.go | 200 +++++++++++++++++++++++++++++++ internal/options/options.go | 97 --------------- internal/x509util/x509util.go | 44 ------- 15 files changed, 947 insertions(+), 632 deletions(-) create mode 100644 internal/feedback/feedback.go create mode 100644 internal/flagopt/flagopt.go create mode 100644 internal/ioutil/ioutil.go create mode 100644 internal/logger/logger.go delete mode 100644 internal/manager/helpers.go delete mode 100644 internal/merkle/TODO delete mode 100644 internal/merkle/compact.go delete mode 100644 internal/merkle/merkle.go create mode 100644 internal/monitor/chunks.go create mode 100644 internal/monitor/matcher.go create mode 100644 internal/monitor/monitor.go create mode 100644 internal/monitor/tail.go delete mode 100644 internal/options/options.go delete mode 100644 internal/x509util/x509util.go (limited to 'internal') diff --git a/internal/feedback/feedback.go b/internal/feedback/feedback.go new file mode 100644 index 0000000..77431e0 --- /dev/null +++ b/internal/feedback/feedback.go @@ -0,0 +1,127 @@ +package feedback + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "time" + + "rgdd.se/silent-ct/internal/logger" + "rgdd.se/silent-ct/pkg/crtutil" + "rgdd.se/silent-ct/pkg/policy" + "rgdd.se/silent-ct/pkg/submission" +) + +type Event struct { + NodeName string // Name of the node that generated a submission + PEMChain []byte // A certificate chain found in the submission +} + +type Config struct { + Policy policy.Policy + + // Optional + Logger logger.Logger // Debug and info prints only (no output by default) + PullInterval time.Duration // How often nodes are pulled via HTTP GET + HTTPTimeout time.Duration // Timeout to use when pulling nodes +} + +type Feedback struct { + cfg Config + nodes []policy.Node + eventCh chan []Event +} + +func New(cfg Config, eventCh chan []Event) (Feedback, error) { + if !cfg.Logger.IsConfigured() { + cfg.Logger = logger.New(logger.Config{Level: logger.LevelNotice, File: os.Stdout}) + } + if cfg.PullInterval == 0 { + cfg.PullInterval = 1 * time.Hour + } + if cfg.HTTPTimeout == 0 { + cfg.HTTPTimeout = 10 * time.Second + } + + for i, node := range cfg.Policy.Nodes { + if err := node.Validate(); err != nil { + return Feedback{}, fmt.Errorf("node %d: %v", i, err) + } + } + return Feedback{cfg: cfg, nodes: cfg.Policy.Nodes, eventCh: eventCh}, nil +} + +// RunForever collects legitimately issued certificates from nodes +func (fb *Feedback) RunForever(ctx context.Context) { + ticker := time.NewTicker(fb.cfg.PullInterval) + defer ticker.Stop() + + fb.pullOnce(ctx) + select { + case <-ticker.C: + fb.pullOnce(ctx) + case <-ctx.Done(): + return + } +} + +func (fb *Feedback) pullOnce(ctx context.Context) { + fb.cfg.Logger.Debugf("pull %d nodes\n", len(fb.nodes)) + for _, node := range fb.nodes { + data, err := fb.pull(ctx, node) + if err != nil { + fb.cfg.Logger.Debugf("failed to pull node %s: %v", node.Name, err) + continue + } + + var events []Event + for _, pemChain := range data { + chain, err := crtutil.CertificateChainFromPEM(pemChain) + if err != nil { + fb.cfg.Logger.Infof("failed to parse certificate from node %s: %v", node.Name, err) + continue + } + if err := node.Authorize(chain[0].DNSNames); err != nil { + fb.cfg.Logger.Infof("%s\n", err.Error()) + continue + } + + events = append(events, Event{NodeName: node.Name, PEMChain: pemChain}) + } + + fb.eventCh <- events + } +} + +func (fb *Feedback) pull(ctx context.Context, node policy.Node) ([][]byte, error) { + req, err := http.NewRequest(http.MethodGet, node.URL, nil) + if err != nil { + return nil, fmt.Errorf("new request: %v", err) + } + req.WithContext(ctx) + + cli := http.Client{Timeout: fb.cfg.HTTPTimeout} + rsp, err := cli.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP %s: %v", req.Method, err) + } + defer rsp.Body.Close() + + b, err := io.ReadAll(rsp.Body) + if err != nil { + return nil, fmt.Errorf("read response: %v", err) + } + if rsp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s: %s", rsp.Status, string(b)) + } + + s := submission.Submission(b) + data, err := s.Open(node) + if err != nil { + return nil, fmt.Errorf("open: %v", err) + } + + return data, nil +} diff --git a/internal/flagopt/flagopt.go b/internal/flagopt/flagopt.go new file mode 100644 index 0000000..484270d --- /dev/null +++ b/internal/flagopt/flagopt.go @@ -0,0 +1,26 @@ +package flagopt + +import ( + "flag" + "time" +) + +func BoolOpt(fs *flag.FlagSet, opt *bool, short, long string, value bool) { + fs.BoolVar(opt, short, value, "") + fs.BoolVar(opt, long, value, "") +} + +func DurationOpt(fs *flag.FlagSet, opt *time.Duration, short, long string, value time.Duration) { + fs.DurationVar(opt, short, value, "") + fs.DurationVar(opt, long, value, "") +} + +func UintOpt(fs *flag.FlagSet, opt *uint, short, long string, value uint) { + fs.UintVar(opt, short, value, "") + fs.UintVar(opt, long, value, "") +} + +func StringOpt(fs *flag.FlagSet, opt *string, short, long, value string) { + fs.StringVar(opt, short, value, "") + fs.StringVar(opt, long, value, "") +} diff --git a/internal/ioutil/ioutil.go b/internal/ioutil/ioutil.go new file mode 100644 index 0000000..7fe6cfc --- /dev/null +++ b/internal/ioutil/ioutil.go @@ -0,0 +1,56 @@ +package ioutil + +import ( + "encoding/json" + "fmt" + "os" +) + +func CommitData(path string, data []byte) error { + return os.WriteFile(path, data, 0644) // FIXME: use safefile package for atomic file writes +} + +func ReadData(path string) ([]byte, error) { + return os.ReadFile(path) +} + +func CommitJSON(path string, obj any) error { + b, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return err + } + return CommitData(path, b) +} + +func ReadJSON(path string, obj any) error { + b, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("%s: %v", path, err) + } + return json.Unmarshal(b, obj) +} + +func CreateDirectories(paths []string) error { + for _, path := range paths { + if err := os.Mkdir(path, 0755); err != nil { + return err + } + } + return nil +} + +func DirectoriesExist(paths []string) error { + for _, path := range paths { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return fmt.Errorf("directory does not exist: %s", path) + } + if err != nil { + return err + } + if !info.IsDir() { + return fmt.Errorf("%s: is not a directory", path) + } + } + return nil +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 0000000..195ad3e --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,96 @@ +package logger + +import ( + "fmt" + "log" + "os" + "sync" +) + +const ( + LevelDebug Level = iota + 1 + LevelInfo + LevelNotice + LevelFatal +) + +type Level int + +func NewLevel(str string) (Level, error) { + switch str { + case LevelDebug.String(): + return LevelDebug, nil + case LevelInfo.String(): + return LevelInfo, nil + case LevelNotice.String(): + return LevelNotice, nil + case LevelFatal.String(): + return LevelFatal, nil + } + return Level(0), fmt.Errorf("unknown level %q", str) +} + +func (lv Level) String() string { + switch lv { + case LevelDebug: + return "DEBUG" + case LevelInfo: + return "INFO" + case LevelNotice: + return "NOTICE" + case LevelFatal: + return "FATAL" + default: + return "UNKNOWN" + } +} + +type Config struct { + Level Level + File *os.File +} + +type Logger struct { + cfg Config + log log.Logger + mutex sync.Mutex +} + +func New(cfg Config) (l Logger) { + l.Reconfigure(cfg) + return +} + +func (l *Logger) Reconfigure(cfg Config) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if cfg.Level < LevelDebug || cfg.Level > LevelFatal { + cfg.Level = LevelNotice + } + if cfg.File == nil { + cfg.File = os.Stdout + } + + l.cfg = cfg + l.log = *log.New(l.cfg.File, "", log.Ldate|log.Ltime) +} + +func (l *Logger) IsConfigured() bool { + return l.cfg.File != nil +} + +func (l *Logger) Debugf(format string, args ...interface{}) { l.printf(LevelDebug, format, args...) } +func (l *Logger) Infof(format string, args ...interface{}) { l.printf(LevelInfo, format, args...) } +func (l *Logger) Noticef(format string, args ...interface{}) { l.printf(LevelNotice, format, args...) } +func (l *Logger) Fatalf(format string, args ...interface{}) { l.printf(LevelFatal, format, args...) } +func (l *Logger) Dief(format string, args ...interface{}) { l.Fatalf(format, args...); os.Exit(1) } + +func (l *Logger) printf(lv Level, format string, args ...interface{}) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.cfg.Level <= lv { + l.log.Printf("["+lv.String()+"] "+format, args...) + } +} diff --git a/internal/manager/helpers.go b/internal/manager/helpers.go deleted file mode 100644 index a9a2158..0000000 --- a/internal/manager/helpers.go +++ /dev/null @@ -1,52 +0,0 @@ -package manager - -import ( - "crypto/sha256" - "encoding/base64" - "fmt" - - ct "github.com/google/certificate-transparency-go" - "gitlab.torproject.org/rgdd/ct/pkg/metadata" - "rgdd.se/silent-ct/pkg/monitor" -) - -func selectLogs(m metadata.Metadata) []monitor.MessageLogConfig { - var logs []monitor.MessageLogConfig - for _, operator := range m.Operators { - for _, log := range operator.Logs { - if log.State == nil { - continue // ignore logs without a state (should not happen) - } - if log.State.Name == metadata.LogStatePending { - continue // log is not yet relevant - } - if log.State.Name == metadata.LogStateRetired { - continue // log is not expected to be reachable - } - if log.State.Name == metadata.LogStateRejected { - continue // log is not expected to be reachable - } - - // FIXME: remove me instead of hard coding Argon 2024 - id, _ := log.Key.ID() - got := fmt.Sprintf("%s", base64.StdEncoding.EncodeToString(id[:])) - want := "7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=" - if got != want { - continue - } - - logs = append(logs, monitor.MessageLogConfig{ - Metadata: log, - State: monitor.MonitorState{ - LogState: monitor.LogState{ct.SignedTreeHead{ - SHA256RootHash: [sha256.Size]byte{47, 66, 110, 15, 246, 154, 8, 100, 150, 140, 206, 208, 17, 57, 112, 116, 210, 3, 19, 55, 46, 63, 209, 12, 234, 130, 225, 124, 237, 2, 64, 228}, - TreeSize: 610650601, - Timestamp: 1702108968538, - }}, - NextIndex: 388452203, - }, - }) - } - } - return logs -} diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 33207e9..bc216c1 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -2,93 +2,208 @@ package manager import ( "context" - "encoding/json" + "errors" "fmt" "os" "time" "gitlab.torproject.org/rgdd/ct/pkg/metadata" - "rgdd.se/silent-ct/pkg/monitor" - "rgdd.se/silent-ct/pkg/server" -) - -const ( - DefaultStateDir = "/home/rgdd/.local/share/silent-ct" // FIXME - DefaultMetadataRefreshInterval = 1 * time.Hour + "rgdd.se/silent-ct/internal/feedback" + "rgdd.se/silent-ct/internal/logger" + "rgdd.se/silent-ct/internal/monitor" + "rgdd.se/silent-ct/pkg/policy" + "rgdd.se/silent-ct/pkg/storage" ) type Config struct { - StateDir string - Nodes server.Nodes + Policy policy.Policy + Bootstrap bool // Whether a new storage should be initialized from scratch + Directory string // Path to a directory where everything will be stored - MetadataRefreshInterval time.Duration + // Optional + Logger logger.Logger // Where to output messages and with what verbosity + AlertDelay time.Duration // Time before alerting on certificates that are unaccounted for + MetadataRefreshInterval time.Duration // How often to update the list of monitored logs + ShutdownTimeout time.Duration // Force shutdown after this timeout (FIXME: should not be needed) } type Manager struct { Config + storage.Storage + + feventCh chan []feedback.Event + meventCh chan monitor.Event + mconfigCh chan monitor.MonitoredLog + errorCh chan error } -func New(cfg Config) (Manager, error) { - if cfg.StateDir == "" { - cfg.StateDir = DefaultStateDir +func New(cfg Config, fch chan []feedback.Event, mch chan monitor.Event, cch chan monitor.MonitoredLog, ech chan error) (Manager, error) { + if !cfg.Logger.IsConfigured() { + cfg.Logger = logger.New(logger.Config{Level: logger.LevelNotice, File: os.Stdout}) } if cfg.MetadataRefreshInterval == 0 { - cfg.MetadataRefreshInterval = DefaultMetadataRefreshInterval + cfg.MetadataRefreshInterval = 1 * time.Hour + } + if cfg.ShutdownTimeout == 0 { + cfg.ShutdownTimeout = 1 * time.Second // FIXME: increase } - return Manager{Config: cfg}, nil -} - -func (mgr *Manager) Run(ctx context.Context, - serverCh chan server.MessageNodeSubmission, - monitorCh chan monitor.MessageLogProgress, - configCh chan []monitor.MessageLogConfig, - errorCh chan error) error { - md, err := mgr.metadataRead() + s, err := storage.New(storage.Config{ + Bootstrap: cfg.Bootstrap, + Directory: cfg.Directory, + AlertDelay: cfg.AlertDelay, + StaticLogs: cfg.Policy.StaticLogs, + RemoveLogs: cfg.Policy.RemoveLogs, + }) if err != nil { - return fmt.Errorf("read metadata: %v\n", err) + return Manager{}, err + } + + for _, log := range s.LogList.Generate() { + state, err := s.BootstrapLog(context.Background(), log, cfg.Bootstrap) + if errors.Is(err, storage.ErrorMonitorStateExists) { + continue + } + if err != nil { + return Manager{}, err + } + cfg.Logger.Infof("bootstrapping log %s at next index %d\n", log.URL, state.NextIndex) + } + + return Manager{Config: cfg, Storage: s, feventCh: fch, meventCh: mch, mconfigCh: cch, errorCh: ech}, nil +} + +func (mgr *Manager) Run(ctx context.Context) error { + if err := mgr.startupConfig(); err != nil { + return fmt.Errorf("unable to do startup config: %v", err) } - configCh <- selectLogs(md) - ticker := time.NewTicker(mgr.MetadataRefreshInterval) - defer ticker.Stop() + metadataTicker := time.NewTicker(mgr.MetadataRefreshInterval) + defer metadataTicker.Stop() + shutdown := false for { select { - case <-ctx.Done(): - return nil - case <-ticker.C: - mu, err := mgr.metadataUpdate(ctx, md) - if err != nil { - continue + case <-metadataTicker.C: + if err := mgr.metadataJob(ctx); err != nil { + return fmt.Errorf("unable to run metadata job: %v", err) + } + case ev := <-mgr.meventCh: + if err := mgr.monitorJob(ev); err != nil { + return fmt.Errorf("unable to run monitor job: %v", err) + } + if err := mgr.alertJob(); err != nil { + return fmt.Errorf("unable to run alert job: %v\n", err) } - if mu.Version.Major <= md.Version.Major { + case ev := <-mgr.feventCh: + if err := mgr.feedbackJob(ev); err != nil { + return fmt.Errorf("unable to run server job: %v", err) + } + if err := mgr.alertJob(); err != nil { + return fmt.Errorf("unable to run alert job: %v\n", err) + } + case err := <-mgr.errorCh: + if err := mgr.errorJob(err); err != nil { + return fmt.Errorf("unable to run error job: %v", err) + } + case <-ctx.Done(): + if !shutdown { + shutdown = true + mgr.Logger.Noticef("shutdown scheduled in %v\n", mgr.ShutdownTimeout) + + // defer shutdown so that all channels can be drained + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), mgr.ShutdownTimeout) + defer cancel() continue } - md = mu - configCh <- selectLogs(md) - case ev := <-monitorCh: - fmt.Printf("DEBUG: received event from monitor with %d matches\n", len(ev.Matches)) - case ev := <-serverCh: - fmt.Printf("DEBUG: received event from server: %v\n", ev) - case err := <-errorCh: - fmt.Printf("DEBUG: received error: %v\n", err) + + mgr.Logger.Debugf("manager shutdown\n") + os.Exit(0) // FIXME: return nil without hanging, unpredictable gh.com/google/ct-go fetcher shutdown? } } } -func (mgr *Manager) metadataRead() (metadata.Metadata, error) { - b, err := os.ReadFile(mgr.StateDir + "/metadata.json") +func (mgr *Manager) startupConfig() error { + mgr.Logger.Debugf("startup configuring contains %d logs\n", len(mgr.Storage.LogList.Generate())) + for _, log := range mgr.Storage.LogList.Generate() { + state, err := mgr.GetMonitorState(log) + if err != nil { + return err + } + mgr.mconfigCh <- monitor.MonitoredLog{Config: log, State: state} + } + return nil +} + +func (mgr *Manager) metadataJob(ctx context.Context) error { + mgr.Logger.Debugf("running metadata job\n") + added, removed, err := mgr.LogList.Update(ctx) + if err != nil { + if mgr.LogList.IsStale() { + return fmt.Errorf("unable to update log list which is now stale: %v", err) + } + } + + mgr.removeLogs(removed) + mgr.addLogs(ctx, added) + return nil +} + +func (mgr *Manager) removeLogs(logs []metadata.Log) { + mgr.Logger.Debugf("removing %d logs\n", len(logs)) + for _, log := range logs { + state, _ := mgr.GetMonitorState(log) + mgr.Logger.Infof("removing log %s with %d entries in its backlog\n", log.URL, state.TreeSize-state.NextIndex) + mgr.mconfigCh <- monitor.MonitoredLog{Config: log} + } +} + +func (mgr *Manager) addLogs(ctx context.Context, logs []metadata.Log) { + mgr.Logger.Debugf("adding %d logs\n", len(logs)) + for _, log := range logs { + state, err := mgr.BootstrapLog(ctx, log, false) + if errors.Is(err, storage.ErrorMonitorStateExists) { + mgr.Logger.Infof("adding log %s with existing state on disk\n", log.URL) + } else if err != nil { + mgr.Logger.Noticef("restart required: failed to bootstrap new log %s: %v\n", log.URL, err) + } else { + mgr.Logger.Infof("bootstrapping log %s at next index 0\n", log.URL) + } + mgr.mconfigCh <- monitor.MonitoredLog{Config: log, State: state} + } +} + +func (mgr *Manager) feedbackJob(events []feedback.Event) error { + mgr.Logger.Debugf("received feedback with %d events", len(events)) + for _, ev := range events { + if err := mgr.AddChain(ev.NodeName, ev.PEMChain); err != nil { + return err + } + } + return nil +} + +func (mgr *Manager) monitorJob(msg monitor.Event) error { + mgr.Logger.Debugf("new state for %s\n", msg.Summary()) + if err := mgr.AddEntries(msg.State.LogID, msg.Matches); err != nil { + return err + } + return mgr.SetMonitorState(msg.State.LogID, msg.State) +} + +func (mgr *Manager) alertJob() error { + alerts, err := mgr.Index.TriggerAlerts() if err != nil { - return metadata.Metadata{}, err + return err } - var md metadata.Metadata - if err := json.Unmarshal(b, &md); err != nil { - return metadata.Metadata{}, err + for _, alert := range alerts { + mgr.Logger.Noticef("certificate mis-issuance? No node submitted certificate %s\n", alert.StoredAt) } - return md, nil + return nil } -func (mgr *Manager) metadataUpdate(ctx context.Context, old metadata.Metadata) (metadata.Metadata, error) { - return metadata.Metadata{}, fmt.Errorf("TODO: update metadata") +func (mgr *Manager) errorJob(err error) error { + mgr.Logger.Debugf("received error: %v\n", err) + return nil } diff --git a/internal/merkle/TODO b/internal/merkle/TODO deleted file mode 100644 index 46cc0cb..0000000 --- a/internal/merkle/TODO +++ /dev/null @@ -1 +0,0 @@ -Drop this package, fix the minor edit in upstream. diff --git a/internal/merkle/compact.go b/internal/merkle/compact.go deleted file mode 100644 index 6eeabd0..0000000 --- a/internal/merkle/compact.go +++ /dev/null @@ -1,115 +0,0 @@ -// BSD 2-Clause License -// -// Copyright (c) 2022, the ct authors -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// From: -// https://gitlab.torproject.org/rgdd/ct/-/tree/main/pkg/merkle -// -// The only difference is that leaf hashes rather than leaf data are passed as -// input to TreeHeadFromRangeProof, thus also changing the nodes() helper. -package merkle - -import ( - "crypto/sha256" - "fmt" -) - -// node represents a subtree at some level and a particular index -type node struct { - index uint64 - hash [sha256.Size]byte -} - -// nodes returns a list of consecutive leaf hashes -func nodes(index uint64, leafHashes [][sha256.Size]byte) (n []node) { - for i, lh := range leafHashes { - n = append(n, node{index + uint64(i), lh}) - } - return -} - -// compactRange outputs the minimal number of fixed subtree hashes given a -// non-empty list of consecutive leaves that start from a non-zero index. For a -// definition of this algorithm, see the end of ../../doc/tlog_algorithms.md. -func compactRange(nodes []node) [][sha256.Size]byte { - // Step 1 - var hashes [][sha256.Size]byte - - // Step 2 - for len(nodes) > 1 { - // Step 2a - if xor(nodes[1].index, 1) != nodes[0].index { - hashes = append(hashes, nodes[0].hash) - nodes = nodes[1:] - } - - // Step 2b; Step 2c; Step 2c(iii) - for i := 0; i < len(nodes); i++ { - // Step 2c(i) - if i+1 != len(nodes) { - nodes[i].hash = HashInteriorNode(nodes[i].hash, nodes[i+1].hash) - nodes = append(nodes[:i+1], nodes[i+2:]...) - } - - // Step 2c(ii) - nodes[i].index = rshift(nodes[i].index) - } - } - - // Step 3 - return append(hashes, nodes[0].hash) -} - -// TreeHeadFromRangeProof computes a tree head at size n=len(leafHashes)+index -// if given a list of leaf hashes at indices index,...,n-1 as well as an -// inclusion proof for the first leaf in the tree of size n. This allows a -// verifier to check inclusion of one or more log entries with a single -// inclusion proof. -func TreeHeadFromRangeProof(leafHashes [][sha256.Size]byte, index uint64, proof [][sha256.Size]byte) (root [sha256.Size]byte, err error) { - var cr [][sha256.Size]byte - confirmHash := func(h [sha256.Size]byte) error { - if h != cr[0] { - return fmt.Errorf("aborted due incorrect right-node subtree hash") - } - cr = cr[1:] - return nil - } - copyRoot := func(r [sha256.Size]byte) error { - root = r - return nil - } - - if len(leafHashes) == 0 { - return [sha256.Size]byte{}, fmt.Errorf("need at least one leaf to recompute tree head from proof") - } - if len(leafHashes) > 1 { - cr = compactRange(nodes(index+1, leafHashes[1:])) - } - return root, inclusion(leafHashes[0], index, index+uint64(len(leafHashes)), proof, copyRoot, confirmHash) -} - -func xor(a, b uint64) uint64 { - return a ^ b -} diff --git a/internal/merkle/merkle.go b/internal/merkle/merkle.go deleted file mode 100644 index 872364f..0000000 --- a/internal/merkle/merkle.go +++ /dev/null @@ -1,271 +0,0 @@ -// BSD 2-Clause License -// -// Copyright (c) 2022, the ct authors -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// From: -// https://gitlab.torproject.org/rgdd/ct/-/tree/main/pkg/merkle -package merkle - -import ( - "crypto/sha256" - "fmt" -) - -// HashEmptyTree computes the hash of an empty tree. See RFC 6162, §2.1: -// -// MTH({}) = SHA-256() -func HashEmptyTree() [sha256.Size]byte { - return sha256.Sum256(nil) -} - -// HashLeafNode computes the hash of a leaf's data. See RFC 6162, §2.1: -// -// MTH({d(0)}) = SHA-256(0x00 || d(0)) -func HashLeafNode(data []byte) (hash [sha256.Size]byte) { - h := sha256.New() - h.Write([]byte{0x00}) - h.Write(data) - copy(hash[:], h.Sum(nil)) - return -} - -// HashInteriorNode computes the hash of an interior node. See RFC 6962, §2.1: -// -// MTH(D[n]) = SHA-256(0x01 || MTH(D[0:k]) || MTH(D[k:n]) -func HashInteriorNode(left, right [sha256.Size]byte) (hash [sha256.Size]byte) { - h := sha256.New() - h.Write([]byte{0x01}) - h.Write(left[:]) - h.Write(right[:]) - copy(hash[:], h.Sum(nil)) - return -} - -// inclusion implements the algorithm specified in RFC 9162, Section 2.1.3.2. -// In addition, the caller is allowed to confirm right-node subtree hashes. -func inclusion(leaf [sha256.Size]byte, index, size uint64, proof [][sha256.Size]byte, - confirmRoot func([sha256.Size]byte) error, confirmHash func([sha256.Size]byte) error) error { - // Step 1 - if index >= size { - return fmt.Errorf("leaf index must be in [%d, %d]", 0, size-1) - } - - // Step 2 - fn := index - sn := size - 1 - - // Step 3 - r := leaf - - // Step 4 - for i, p := range proof { - // Step 4a - if sn == 0 { - return fmt.Errorf("reached tree head with %d remaining proof hash(es)", len(proof[i:])) - } - - // Step 4b - if isLSB(fn) || fn == sn { - // Step 4b, i - r = HashInteriorNode(p, r) - - // Step 4b, ii - if !isLSB(fn) { - for { - fn = rshift(fn) - sn = rshift(sn) - - if isLSB(fn) || fn == 0 { - break - } - } - } - } else { - // Step 4b, i - r = HashInteriorNode(r, p) - - // Extension: allow the caller to confirm right-node subtree hashes - if err := confirmHash(p); err != nil { - return fmt.Errorf("subtree index %d: %v", fn, err) - } - } - - // Step 4c - fn = rshift(fn) - sn = rshift(sn) - } - - // Step 5 - if sn != 0 { - return fmt.Errorf("stopped at subtree with index %d due to missing proof hashes", fn) - } - return confirmRoot(r) -} - -// consistency implements the algorithm specified in RFC 9162, §2.1.4.2 -func consistency(oldSize, newSize uint64, oldRoot, newRoot [sha256.Size]byte, proof [][sha256.Size]byte) error { - // Step 1 - if len(proof) == 0 { - return fmt.Errorf("need at least one proof hash") - } - - // Step 2 - if isPOW2(oldSize) { - proof = append([][sha256.Size]byte{oldRoot}, proof...) - } - - // Step 3 - fn := oldSize - 1 - sn := newSize - 1 - - // Step 4 - for isLSB(fn) { - fn = rshift(fn) - sn = rshift(sn) - } - - // Step 5 - fr := proof[0] - sr := proof[0] - - // Step 6 - for i, c := range proof[1:] { - // Step 6a - if sn == 0 { - return fmt.Errorf("reached tree head with %d remaining proof hash(es)", len(proof[i+1:])) - } - - // Step 6b - if isLSB(fn) || fn == sn { - // Step 6b, i - fr = HashInteriorNode(c, fr) - // Step 6b, ii - sr = HashInteriorNode(c, sr) - // Step 6b, iii - if !isLSB(fn) { - for { - fn = rshift(fn) - sn = rshift(sn) - - if isLSB(fn) || fn == 0 { - break - } - } - } - } else { - // Step 6b, i - sr = HashInteriorNode(sr, c) - } - - // Step 6c - fn = rshift(fn) - sn = rshift(sn) - } - - // Step 7 - if sn != 0 { - return fmt.Errorf("stopped at subtree with index %d due to missing proof hashes", fn) - } - if fr != oldRoot { - return fmt.Errorf("recomputed old tree head %x is not equal to reference tree head %x", fr[:], oldRoot[:]) - } - if sr != newRoot { - return fmt.Errorf("recomputed new tree head %x is not equal to reference tree head %x", sr[:], newRoot[:]) - } - return nil -} - -// VerifyInclusion verifies that a leaf's data is commited at a given index in a -// reference tree -func VerifyInclusion(data []byte, index, size uint64, root [sha256.Size]byte, proof [][sha256.Size]byte) error { - if size == 0 { - return fmt.Errorf("tree size must be larger than zero") - } - - confirmHash := func(h [sha256.Size]byte) error { return nil } // No compact range extension - confirmRoot := func(r [sha256.Size]byte) error { - if r != root { - return fmt.Errorf("recomputed tree head %x is not equal to reference tree head %x", r[:], root[:]) - } - return nil - } - return inclusion(HashLeafNode(data), index, size, proof, confirmRoot, confirmHash) -} - -// VerifyConsistency verifies that an an old tree is consistent with a new tree -func VerifyConsistency(oldSize, newSize uint64, oldRoot, newRoot [sha256.Size]byte, proof [][sha256.Size]byte) error { - checkTree := func(size uint64, root [sha256.Size]byte) error { - if size == 0 { - if root != HashEmptyTree() { - return fmt.Errorf("non-empty tree head %x for size zero", root[:]) - } - if len(proof) != 0 { - return fmt.Errorf("non-empty proof with %d hashes for size zero", len(proof)) - } - } else if root == HashEmptyTree() { - return fmt.Errorf("empty tree head %x for tree size %d", root[:], size) - } - return nil - } - - if err := checkTree(oldSize, oldRoot); err != nil { - return fmt.Errorf("old: %v", err) - } - if err := checkTree(newSize, newRoot); err != nil { - return fmt.Errorf("new: %v", err) - } - if oldSize == 0 { - return nil - } - - if oldSize == newSize { - if oldRoot != newRoot { - return fmt.Errorf("different tree heads %x and %x with equal tree size %d", oldRoot, newRoot, oldSize) - } - if len(proof) != 0 { - return fmt.Errorf("non-empty proof with %d hashes for equal tree size %d", len(proof), oldSize) - } - return nil - } - if oldSize > newSize { - return fmt.Errorf("old tree size %d must be smaller than or equal to the new tree size %d", oldSize, newSize) - } - - return consistency(oldSize, newSize, oldRoot, newRoot, proof) -} - -// isLSB returns true if the least significant bit of num is set -func isLSB(num uint64) bool { - return (num & 1) != 0 -} - -// isPOW2 returns true if num is a power of two (1, 2, 4, 8, ...) -func isPOW2(num uint64) bool { - return (num & (num - 1)) == 0 -} - -func rshift(num uint64) uint64 { - return num >> 1 -} diff --git a/internal/monitor/chunks.go b/internal/monitor/chunks.go new file mode 100644 index 0000000..02b3802 --- /dev/null +++ b/internal/monitor/chunks.go @@ -0,0 +1,89 @@ +package monitor + +// +// A min heap of chunks, ordered on each chunk's start index. This makes it +// easy to order the downloaded leaves when using multiple parallell fetchers. +// +// Credit: inspiration to use a heap from Aaron Gable, see +// https://github.com/aarongable/ctaudit +// + +import ( + "container/heap" + "crypto/sha256" +) + +type chunk struct { + startIndex uint64 // Index of the first leaf + leafHashes [][sha256.Size]byte // List of consecutive leaf hashes + matches []LogEntry // Leaves that match some criteria + errors []error // Errors that ocurred while matching on the leaves +} + +type chunks []*chunk + +func newChunks() *chunks { + var h chunks + heap.Init((*internal)(&h)) + return &h +} + +func (h *chunks) push(c *chunk) { + heap.Push((*internal)(h), c) +} + +func (h *chunks) pop() *chunk { + x := heap.Pop((*internal)(h)) + return x.(*chunk) +} + +// gap returns true if there's a gap between the provided start index and the +// top most chunk. If the top most chunk is in sequence, it is merged with +// any following chunks that are also in sequence to form one larger chunk. +func (h *chunks) gap(start uint64) bool { + if len(*h) == 0 { + return true + } + + top := h.pop() + if start != top.startIndex { + h.push(top) + return true + } + + for len(*h) > 0 { + c := h.pop() + if c.startIndex != top.startIndex+uint64(len(top.leafHashes)) { + h.push(c) + break + } + + top.leafHashes = append(top.leafHashes, c.leafHashes...) + top.matches = append(top.matches, c.matches...) + top.errors = append(top.errors, c.errors...) + } + + h.push(top) + return false +} + +// internal implements the heap interface, see example: +// https://cs.opensource.google/go/go/+/refs/tags/go1.21.5:src/container/heap/example_intheap_test.go +type internal chunks + +func (h internal) Len() int { return len(h) } +func (h internal) Less(i, j int) bool { return h[i].startIndex < h[j].startIndex } +func (h internal) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *internal) Push(x any) { + *h = append(*h, x.(*chunk)) +} + +func (h *internal) Pop() any { + old := *h + n := len(old) + x := old[n-1] + old[n-1] = nil // avoid memory leak + *h = old[:n-1] + return x +} diff --git a/internal/monitor/matcher.go b/internal/monitor/matcher.go new file mode 100644 index 0000000..912e595 --- /dev/null +++ b/internal/monitor/matcher.go @@ -0,0 +1,13 @@ +package monitor + +type Matcher interface { + // Match determines if a log entry is considered to be a "match" based on + // some criteria. An error is returned if any certificate parsing fails. + Match(leafInput, extraData []byte) (bool, error) +} + +type MatchAll struct{} + +func (m *MatchAll) Match(leafInput, extraData []byte) (bool, error) { + return true, nil +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go new file mode 100644 index 0000000..6accd97 --- /dev/null +++ b/internal/monitor/monitor.go @@ -0,0 +1,173 @@ +// Package monitor provides monitoring of Certificate Transparency logs. If +// running in continuous mode, the list of logs can be updated dynamically. +// +// Implement the Matcher interface to customize which certificates should be +// included in the monitor's emitted events. See MatchAll for an example. +// +// Note that this package verifies that the monitored logs are locally +// consistent with regard to the initial start-up state. It is up to the user +// to process the monitor's emitted events and errors, and to persist state. +package monitor + +import ( + "context" + "crypto/x509" + "encoding/base64" + "fmt" + "net/http" + "os" + "sync" + + ct "github.com/google/certificate-transparency-go" + "github.com/google/certificate-transparency-go/client" + "github.com/google/certificate-transparency-go/jsonclient" + "gitlab.torproject.org/rgdd/ct/pkg/metadata" + "rgdd.se/silent-ct/internal/logger" +) + +// MonitoredLog provides information about a log the monitor is following +type MonitoredLog struct { + Config metadata.Log + State State +} + +// State is the latest append-only state the monitor observed from its local +// vantage point. The next entry to download is specified by NextIndex. +type State struct { + ct.SignedTreeHead `json:"latest_sth"` + NextIndex uint64 `json:"next_index"` +} + +// Event carries the latest consistent monitor state, found matches, as well as +// errors that occurred while trying to match on the downloaded log entries. +type Event struct { + State State + Matches []LogEntry + Errors []error +} + +func (ev *Event) Summary() string { + return fmt.Sprintf("log %s: tree size %d at next index %d (%d matches, %d errors)", + base64.StdEncoding.EncodeToString(ev.State.LogID[:]), + ev.State.TreeSize, ev.State.NextIndex, len(ev.Matches), len(ev.Errors)) +} + +// LogEntry is a Merkle tree leaf in a log +type LogEntry struct { + LeafIndex uint64 `json:"leaf_index"` + LeafData []byte `json:"leaf_data"` + ExtraData []byte `json:"extra_data"` +} + +type Config struct { + // Optional + Matcher Matcher // Which log entries to match (default is to match all) + Logger logger.Logger // Debug prints only (no output by default) + Contact string // Something that help log operators get in touch + ChunkSize uint // Min number of leaves to propagate a chunk without matches + BatchSize uint // Max number of certificates to accept per worker + NumWorkers uint // Number of parallel workers to use for each log +} + +type Monitor struct { + cfg Config + matcher Matcher + + eventCh chan Event + configCh chan MonitoredLog + errorCh chan error +} + +func New(cfg Config, evCh chan Event, cfgCh chan MonitoredLog, errCh chan error) (Monitor, error) { + if cfg.Matcher == nil { + cfg.Matcher = &MatchAll{} + } + if !cfg.Logger.IsConfigured() { + cfg.Logger = logger.New(logger.Config{Level: logger.LevelNotice, File: os.Stderr}) + } + if cfg.Contact == "" { + cfg.Contact = "unknown-user" + } + if cfg.ChunkSize == 0 { + cfg.ChunkSize = 256 // FIXME: 16364 + } + if cfg.BatchSize == 0 { + cfg.BatchSize = 1024 + } + if cfg.NumWorkers == 0 { + cfg.NumWorkers = 2 + } + return Monitor{cfg: cfg, matcher: cfg.Matcher, eventCh: evCh, configCh: cfgCh, errorCh: errCh}, nil +} + +func (mon *Monitor) RunOnce(ctx context.Context, cfg []MonitoredLog, evCh chan Event, errCh chan error) error { + return fmt.Errorf("TODO") +} + +func (mon *Monitor) RunForever(ctx context.Context) error { + var wg sync.WaitGroup + defer wg.Wait() + + mctx, cancel := context.WithCancel(ctx) + defer cancel() + + monitoring := make(map[metadata.LogURL]context.CancelFunc) + for { + select { + case <-ctx.Done(): + return nil + case log := <-mon.configCh: + if tcancel, ok := monitoring[log.Config.URL]; ok { + delete(monitoring, log.Config.URL) + tcancel() + continue + } + + newTail := mon.newTailRFC6962 + if log.Config.DNS != nil { // FIXME: get a real nob for tile-based logs + newTail = mon.newTailTile + } + t, err := newTail(log) + if err != nil { + return err + } + + tctx, tcancel := context.WithCancel(mctx) + monitoring[log.Config.URL] = tcancel + + wg.Add(1) + go func(log MonitoredLog, t tail) { + defer wg.Done() + defer tcancel() + t.run(tctx, log, mon.eventCh, mon.errorCh) + }(log, t) + } + } +} + +const userAgentPrefix = "rgdd.se/silent-ct" + +func (mon *Monitor) newTailRFC6962(log MonitoredLog) (tail, error) { + key, err := x509.MarshalPKIXPublicKey(log.Config.Key.Public) + if err != nil { + return tail{}, err + } + cli, err := client.New(string(log.Config.URL), &http.Client{}, jsonclient.Options{ + Logger: &discard{}, + UserAgent: userAgentPrefix + "/" + mon.cfg.Contact, + PublicKeyDER: key, + }) + if err != nil { + return tail{}, err + } + + return tail{cfg: mon.cfg, scanner: cli, checker: cli, matcher: mon.matcher}, nil +} + +func (mon *Monitor) newTailTile(cfg MonitoredLog) (tail, error) { + return tail{}, fmt.Errorf("TODO") +} + +type discard struct{} + +func (n *discard) Printf(string, ...interface{}) {} diff --git a/internal/monitor/tail.go b/internal/monitor/tail.go new file mode 100644 index 0000000..0e16476 --- /dev/null +++ b/internal/monitor/tail.go @@ -0,0 +1,200 @@ +package monitor + +import ( + "context" + "crypto/sha256" + "fmt" + "sync" + "time" + + ct "github.com/google/certificate-transparency-go" + "github.com/google/certificate-transparency-go/client" + "github.com/google/certificate-transparency-go/scanner" + "gitlab.torproject.org/rgdd/ct/pkg/merkle" +) + +type tail struct { + cfg Config + matcher Matcher + scanner scanner.LogClient + checker client.CheckLogClient +} + +func (t *tail) run(ctx context.Context, mon MonitoredLog, eventCh chan Event, errorCh chan error) { + chunkCh := make(chan *chunk) + defer close(chunkCh) + + mctx, cancel := context.WithCancel(ctx) + defer cancel() + + var wg sync.WaitGroup + defer wg.Wait() + + callback := func(eb scanner.EntryBatch) { + c := chunk{startIndex: uint64(eb.Start)} + for i := 0; i < len(eb.Entries); i++ { + c.leafHashes = append(c.leafHashes, merkle.HashLeafNode(eb.Entries[i].LeafInput)) + match, err := t.matcher.Match(eb.Entries[i].LeafInput, eb.Entries[i].ExtraData) + if err != nil { + c.errors = append(c.errors, fmt.Errorf("while processing index %d for %s: %v", i, mon.Config.URL, err)) + continue + } + if !match { + continue + } + + c.matches = append(c.matches, LogEntry{ + LeafIndex: c.startIndex + uint64(i), + LeafData: eb.Entries[i].LeafInput, + ExtraData: eb.Entries[i].ExtraData, + }) + } + + chunkCh <- &c + } + + fetcher := scanner.NewFetcher(t.scanner, &scanner.FetcherOptions{ + BatchSize: int(t.cfg.BatchSize), + StartIndex: int64(mon.State.NextIndex), + ParallelFetch: int(t.cfg.NumWorkers), + Continuous: true, // FIXME: don't set this for read-only log + }) + + wg.Add(1) + go func() { + defer wg.Done() + defer cancel() + fetcher.Run(mctx, callback) + }() + + wg.Add(1) + go func() { + defer wg.Done() + defer cancel() + t.sequence(mctx, mon, eventCh, errorCh, chunkCh) + }() +} + +func (t *tail) sequence(ctx context.Context, mon MonitoredLog, eventCh chan Event, errorCh chan error, chunkCh chan *chunk) { + state := mon.State + heap := newChunks() + for { + select { + case <-ctx.Done(): + return // FIXME: check if we can pop something before return + case c := <-chunkCh: + heap.push(c) + if heap.gap(state.NextIndex) { + continue + } + c = heap.pop() + if len(c.matches) == 0 && len(c.leafHashes) < int(t.cfg.ChunkSize) { + heap.push(c) + continue // FIXME: don't trigger if we havn't run nextState for too long + } + nextState, err := t.nextState(ctx, state, c) + if err != nil { + errorCh <- err + heap.push(c) + continue + } + + state = nextState + eventCh <- Event{State: state, Matches: c.matches, Errors: c.errors} + } + } +} + +func (t *tail) nextState(ctx context.Context, state State, c *chunk) (State, error) { + newState, err := t.nextConsistentState(ctx, state) + if err != nil { + return State{}, err + } + newState, err = t.nextIncludedState(ctx, newState, c) + if err != nil { + return State{}, err + } + return newState, nil +} + +func (t *tail) nextConsistentState(ctx context.Context, state State) (State, error) { + sth, err := getSignedTreeHead(ctx, t.checker) + if err != nil { + return State{}, fmt.Errorf("%s: get-sth: %v", t.checker.BaseURI(), err) + } + sth.LogID = state.SignedTreeHead.LogID + oldSize := state.TreeSize + oldRoot := state.SHA256RootHash + newSize := sth.TreeSize + newRoot := sth.SHA256RootHash + + proof, err := getConsistencyProof(ctx, t.checker, oldSize, newSize) + if err != nil { + return State{}, fmt.Errorf("%s: get-consistency: %v", t.checker.BaseURI(), err) + } + if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(proof)); err != nil { + return State{}, fmt.Errorf("%s: verify consistency: %v", t.checker.BaseURI(), err) + } + + return State{SignedTreeHead: *sth, NextIndex: state.NextIndex}, nil +} + +func (t *tail) nextIncludedState(ctx context.Context, state State, c *chunk) (State, error) { + leafHash := c.leafHashes[0] + oldSize := state.NextIndex + uint64(len(c.leafHashes)) + iproof, err := getInclusionProof(ctx, t.checker, leafHash, oldSize) // FIXME: set leaf index in ctx to hack into tile API + if err != nil { + return State{}, fmt.Errorf("%s: get-inclusion: %v", t.checker.BaseURI(), err) + } + if got, want := uint64(iproof.LeafIndex), state.NextIndex; got != want { + return State{}, fmt.Errorf("%s: wrong index for get-inclusion proof query %x:%d", t.checker.BaseURI(), leafHash[:], oldSize) + } + oldRoot, err := merkle.TreeHeadFromRangeProof(c.leafHashes, state.NextIndex, unslice(iproof.AuditPath)) + if err != nil { + return State{}, fmt.Errorf("%s: range proof: %v", t.checker.BaseURI(), err) + } + + newSize := state.TreeSize + newRoot := state.SHA256RootHash + cproof, err := getConsistencyProof(ctx, t.checker, oldSize, newSize) + if err != nil { + return State{}, fmt.Errorf("%s: get-consistency: %v", t.checker.BaseURI(), err) + } + if err := merkle.VerifyConsistency(oldSize, newSize, oldRoot, newRoot, unslice(cproof)); err != nil { + return State{}, fmt.Errorf("%s: verify consistency: %v", t.checker.BaseURI(), err) + } + + state.NextIndex += uint64(len(c.leafHashes)) + return state, nil +} + +func getInclusionProof(ctx context.Context, cli client.CheckLogClient, leafHash [sha256.Size]byte, size uint64) (*ct.GetProofByHashResponse, error) { + rctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return cli.GetProofByHash(rctx, leafHash[:], size) +} + +func getConsistencyProof(ctx context.Context, cli client.CheckLogClient, oldSize, newSize uint64) ([][]byte, error) { + if oldSize == 0 || oldSize >= newSize { + return [][]byte{}, nil + } + rctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return cli.GetSTHConsistency(rctx, oldSize, newSize) +} + +func getSignedTreeHead(ctx context.Context, cli client.CheckLogClient) (*ct.SignedTreeHead, error) { + rctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return cli.GetSTH(rctx) +} + +func unslice(hashes [][]byte) [][sha256.Size]byte { + var ret [][sha256.Size]byte + for _, hash := range hashes { + var h [sha256.Size]byte + copy(h[:], hash) + ret = append(ret, h) + } + return ret +} diff --git a/internal/options/options.go b/internal/options/options.go deleted file mode 100644 index 3e253c5..0000000 --- a/internal/options/options.go +++ /dev/null @@ -1,97 +0,0 @@ -package options - -import ( - "encoding/json" - "flag" - "fmt" - "os" - - "rgdd.se/silent-ct/internal/manager" - "rgdd.se/silent-ct/pkg/monitor" - "rgdd.se/silent-ct/pkg/server" -) - -const usage = `Usage: - - silent-ct [Options] - -Options: - - -h, --help: Output usage message and exit - -c, --config: Path to a configuration file (Default: %s) - -l, --listen: Listen address to receive submission on (Default: %s) - -s, --state: Path to a directory where state is stored (Default: %s) - -Example configuration file: - - { - "monitor": [ - { - "wildcard": "example.org", - "excludes": [ - "test" - ] - } - ], - "nodes": [ - { - "name": "node_a", - "secret": "aaaa", - "issues": [ - "example.org", - "www.example.org" - ] - } - ] - } - -` - -// Options are command-line options the user can specify -type Options struct { - ListenAddr string - ConfigFile string - StateDir string -} - -func New(cmd string, args []string) (opts Options, err error) { - fs := flag.NewFlagSet(cmd, flag.ContinueOnError) - fs.Usage = func() { - fmt.Fprintf(os.Stderr, usage, server.DefaultConfigFile, server.DefaultAddress, manager.DefaultStateDir) - } - stringOpt(fs, &opts.ConfigFile, "config", "c", server.DefaultConfigFile) - stringOpt(fs, &opts.ListenAddr, "listen", "l", server.DefaultAddress) - stringOpt(fs, &opts.StateDir, "state", "s", manager.DefaultStateDir) - if err = fs.Parse(args); err != nil { - return opts, err - } - - if opts.ConfigFile == "" { - return opts, fmt.Errorf("-c, --config: must not be an empty string") - } - if opts.StateDir == "" { - return opts, fmt.Errorf("-s, --state: must not be an empty string") - } - if opts.ListenAddr == "" { - return opts, fmt.Errorf("-l, --listen: must not be an empty string") - } - return opts, err -} - -func stringOpt(fs *flag.FlagSet, opt *string, short, long, value string) { - fs.StringVar(opt, short, value, "") - fs.StringVar(opt, long, value, "") -} - -type Config struct { - Monitor monitor.MatchWildcards `json:"monitor"` - Nodes server.Nodes `json:"nodes"` -} - -func (c *Config) FromFile(fileName string) error { - b, err := os.ReadFile(fileName) - if err != nil { - return err - } - return json.Unmarshal(b, c) -} diff --git a/internal/x509util/x509util.go b/internal/x509util/x509util.go deleted file mode 100644 index 912d1b4..0000000 --- a/internal/x509util/x509util.go +++ /dev/null @@ -1,44 +0,0 @@ -package x509util - -import ( - "crypto/x509" - "encoding/pem" - "fmt" -) - -// ParseChain parses a certificate chain in PEM format. At least one -// certificate must be in the chain. The first certificate must be a leaf, -// whereas all other certificates must CA certificates (intermdiates/roots). -// -// Note: it is not checked if the certificate chain's root is trusted or not. -func ParseChain(b []byte) ([]x509.Certificate, error) { - var chain []x509.Certificate - - for { - block, rest := pem.Decode(b) - if block == nil { - break - } - crt, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("parse certificate: %v", err) - } - - chain = append(chain, *crt) - b = rest - } - - if len(chain) == 0 { - return nil, fmt.Errorf("no certificates in the provided chain") - } - if chain[0].IsCA { - return nil, fmt.Errorf("leaf certificate has the CA bit set") - } - for _, crt := range chain[1:] { - if !crt.IsCA { - return nil, fmt.Errorf("non-leaf certificate without the CA bit set") - } - } - - return chain, nil -} -- cgit v1.2.3