// Package loglist manages a list of logs to monitor. The list of logs is based // on Google's signed list. Logs can also be added and removed manually. package loglist import ( "context" "fmt" "time" "gitlab.torproject.org/rgdd/ct/pkg/metadata" "rgdd.se/silent-ct/internal/ioutil" ) type Config struct { PermitBootstrap bool // Get and create initial log metadata if nothing valid is available on disk MetadataFile string // Path to a dynamically updated metadata file that can be read/written HistoryDirectory string // Existing directory to store the history of downloaded metadata // Optional MetadataInFuture time.Duration // How wrong the metadata timestamp is allowed to be wrt. future dating MetadataGetsStale time.Duration // How long until the metadata is considered stale MetadataIsRecent time.Duration // How long the metadata is considered recent HTTPTimeout time.Duration // Timeout when fetching metadata StaticLogs []metadata.Log // Takes precedence over the dynamically downloaded metadata RemoveLogs []metadata.LogKey // Logs in the downloaded metadata with these keys are ignored } type LogList struct { cfg Config md metadata.Metadata source metadata.Loader } func New(cfg Config) (LogList, error) { if cfg.MetadataInFuture == 0 { cfg.MetadataInFuture = 24 * time.Hour } if cfg.MetadataGetsStale == 0 { cfg.MetadataGetsStale = 30 * 24 * time.Hour } if cfg.MetadataIsRecent == 0 { cfg.MetadataIsRecent = 1 * time.Hour } if cfg.HTTPTimeout == 0 { cfg.HTTPTimeout = 10 * time.Second } for i, log := range cfg.StaticLogs { if err := checkLog(log); err != nil { return LogList{}, fmt.Errorf("static logs: index %d: %v", i, err) } } for i, key := range cfg.RemoveLogs { if _, err := key.ID(); err != nil { return LogList{}, fmt.Errorf("remove logs: index %d: %v", i, err) } } s := metadata.NewHTTPSource(metadata.HTTPSourceOptions{Name: "google"}) ll := LogList{cfg: cfg, source: &s} if err := ioutil.DirectoriesExist([]string{cfg.HistoryDirectory}); err != nil { return LogList{}, err } if err := ioutil.ReadJSON(cfg.MetadataFile, &ll.md); err != nil { if !ll.cfg.PermitBootstrap { return LogList{}, err } if _, _, err := ll.Update(context.Background()); err != nil { return LogList{}, err } } return ll, nil } func (ll *LogList) IsRecent() bool { return time.Now().Before(ll.md.CreatedAt.Add(ll.cfg.MetadataIsRecent)) } func (ll *LogList) IsStale() bool { return time.Now().After(ll.md.CreatedAt.Add(ll.cfg.MetadataGetsStale)) } func (ll *LogList) Generate() []metadata.Log { var configure []metadata.Log for _, log := range ll.cfg.StaticLogs { configure = append(configure, log) } for _, operator := range ll.md.Operators { for _, log := range operator.Logs { if findKey(ll.cfg.RemoveLogs, log) { continue // static configuration says to remove this log } if findLog(configure, log) { continue // static configuration takes precedence } if skipLog(log) { continue // not in a state where it makes sense to monitor } configure = append(configure, log) } } return configure } func (ll *LogList) Update(ctx context.Context) (added []metadata.Log, removed []metadata.Log, err error) { b, md, err := ll.archiveDynamic(ctx) if err != nil { return } if err = ioutil.CommitData(ll.cfg.MetadataFile, b); err != nil { return } added, removed = metadataLogDiff(ll.md, md) ll.md = md return } func (ll *LogList) archiveDynamic(ctx context.Context) ([]byte, metadata.Metadata, error) { llctx, cancel := context.WithTimeout(ctx, ll.cfg.HTTPTimeout) defer cancel() msg, sig, md, err := ll.source.Load(llctx) if err != nil { return nil, metadata.Metadata{}, err } if future := time.Now().Add(ll.cfg.MetadataInFuture); md.CreatedAt.After(future) { return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is in the future", md.CreatedAt) } if md.CreatedAt.Before(ll.md.CreatedAt) { return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is older than the current list", md.CreatedAt) } // FIXME: consider only archiving on major version bumps path := fmt.Sprintf("%s/%s.json", ll.cfg.HistoryDirectory, time.Now().Format("2006-01-02_1504")) if err := ioutil.CommitData(path, msg); err != nil { return nil, metadata.Metadata{}, err } if err := ioutil.CommitData(path+".sig", sig); err != nil { return nil, metadata.Metadata{}, err } return msg, md, nil }