// Package storage manages an index of certificates, a dynamically updated log // list, and a monitor's state on the local file system in a single directory. package storage import ( "context" "crypto/sha256" "crypto/x509" "fmt" "net/http" "os" "path/filepath" "time" ct "github.com/google/certificate-transparency-go" "github.com/google/certificate-transparency-go/client" "github.com/google/certificate-transparency-go/jsonclient" "github.com/transparency-dev/merkle/compact" "gitlab.torproject.org/rgdd/ct/pkg/metadata" "rgdd.se/silent-ct/internal/ioutil" "rgdd.se/silent-ct/internal/logger" "rgdd.se/silent-ct/internal/logutil" "rgdd.se/silent-ct/internal/monitor" "rgdd.se/silent-ct/pkg/storage/index" "rgdd.se/silent-ct/pkg/storage/loglist" ) type Config struct { Bootstrap bool // Whether a new storage should be bootstrapped in a non-existing directory Directory string // Path to a directory where everything will be stored // Optional Logger *logger.Logger // Info prints only (no output by default) BootstrapTries uint // The number of times to try bootstrapping a log before giving up BootstrapWait time.Duration // How long to wait on bootstrap failures before trying again AlertDelay time.Duration // Time before alerting on certificates that are unaccounted for StaticLogs []metadata.Log // Static logs to configure in loglist RemoveLogs []metadata.LogKey // Keys of logs to omit in loglist } func (cfg *Config) CertificateIndexFile() string { return cfg.Directory + "/crt_index.json" } func (cfg *Config) LegitimateCertificateDirectory() string { return cfg.Directory + "/crt_trusted" } func (cfg *Config) DiscoveredCertificateDirectory() string { return cfg.Directory + "/crt_found" } func (cfg *Config) MetadataFile() string { return cfg.Directory + "/metadata.json" } func (cfg *Config) MetadataHistoryDirectory() string { return cfg.Directory + "/metadata_history" } func (cfg *Config) MonitorStateDirectory() string { return cfg.Directory + "/monitor_state" } func (cfg *Config) MonitorStateFile(logID [sha256.Size]byte) string { return fmt.Sprintf("%s/%x.json", cfg.MonitorStateDirectory(), logID[:]) } func (cfg *Config) directories() []string { return []string{ cfg.Directory, cfg.LegitimateCertificateDirectory(), cfg.DiscoveredCertificateDirectory(), cfg.MetadataHistoryDirectory(), cfg.MonitorStateDirectory(), } } func (cfg *Config) configure() error { if cfg.Directory == "" { return fmt.Errorf("directory is required") } if !cfg.Logger.IsConfigured() { cfg.Logger = logger.New(logger.Config{Level: logger.LevelNotice, File: os.Stderr}) } if cfg.BootstrapTries == 0 { cfg.BootstrapTries = 64 } if cfg.BootstrapWait == 0 { cfg.BootstrapWait = 16 * time.Second } path, err := filepath.Abs(cfg.Directory) if err != nil { return err } cfg.Directory = path if err := ioutil.DirectoriesExist(cfg.directories()); err != nil { if !cfg.Bootstrap { return err } return ioutil.CreateDirectories(cfg.directories()) } return nil } type Storage struct { Config index.Index loglist.LogList } func New(cfg Config) (Storage, error) { err := cfg.configure() if err != nil { return Storage{}, err } s := Storage{Config: cfg} if s.Index, err = index.New(index.Config{ PermitBootstrap: cfg.Bootstrap, IndexFile: cfg.CertificateIndexFile(), TrustDirectory: cfg.LegitimateCertificateDirectory(), MatchDirectory: cfg.DiscoveredCertificateDirectory(), AlertDelay: cfg.AlertDelay, }); err != nil { return Storage{}, err } if s.LogList, err = loglist.New(loglist.Config{ PermitBootstrap: cfg.Bootstrap, MetadataFile: cfg.MetadataFile(), HistoryDirectory: cfg.MetadataHistoryDirectory(), StaticLogs: cfg.StaticLogs, RemoveLogs: cfg.RemoveLogs, }); err != nil { return Storage{}, err } return s, err } func (s *Storage) BootstrapLog(ctx context.Context, log metadata.Log, skipBacklog bool) (monitor.State, error) { if storedState, err := s.GetMonitorState(log); err == nil { return storedState, ErrorMonitorStateExists } cr, sth, err := s.bootstrapWithRetries(ctx, log) if err != nil { return monitor.State{}, err } id, err := log.Key.ID() if err != nil { return monitor.State{}, fmt.Errorf("%s: %v", log.URL, err) } sth.LogID = id state := monitor.State{SignedTreeHead: *sth} if skipBacklog { state.CompactRange = ioutil.UnsliceHashes(cr.Hashes()) state.NextIndex = sth.TreeSize } return state, s.SetMonitorState(id, state) } func (s *Storage) SetMonitorState(logID [sha256.Size]byte, state monitor.State) error { return ioutil.CommitJSON(s.MonitorStateFile(logID), state) } func (s *Storage) GetMonitorState(log metadata.Log) (monitor.State, error) { id, err := log.Key.ID() if err != nil { return monitor.State{}, err } state := monitor.State{} return state, ioutil.ReadJSON(s.MonitorStateFile(id), &state) } func (s *Storage) bootstrapWithRetries(ctx context.Context, log metadata.Log) (*compact.Range, *ct.SignedTreeHead, error) { for tries := s.BootstrapTries; tries > 0; tries-- { cr, sth, err := s.bootstrap(ctx, log) if err == nil { return cr, sth, nil } if tries != 1 { s.Logger.Infof("bootstrap: %v (retry in %v)\n", err, s.BootstrapWait) time.Sleep(s.BootstrapWait) } } return nil, nil, fmt.Errorf("failed to bootstrap after %d attempts: %s", s.BootstrapTries, log.URL) } func (s *Storage) bootstrap(ctx context.Context, log metadata.Log) (*compact.Range, *ct.SignedTreeHead, error) { key, err := x509.MarshalPKIXPublicKey(log.Key.Public) if err != nil { return nil, nil, err } cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{PublicKeyDER: key}) if err != nil { return nil, nil, err } sth, err := logutil.GetSignedTreeHead(ctx, cli) if err != nil { return nil, nil, fmt.Errorf("%s: get-sth: %v", log.URL, err) } entry, err := logutil.GetEntry(ctx, cli, sth.TreeSize-1) if err != nil { return nil, nil, fmt.Errorf("%s: get-entry: %v", log.URL, err) } cr, err := logutil.GetCompactRange(ctx, cli, entry, sth.TreeSize-1) if err != nil { return nil, nil, fmt.Errorf("%s: %v", log.URL, err) } rootHash := logutil.RootHash(cr) if got, want := rootHash, sth.SHA256RootHash; got != want { return nil, nil, fmt.Errorf("invalid root hash %x (want %x)", rootHash[:], sth.SHA256RootHash[:]) } return cr, sth, nil }