1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
// Package loglist manages a list of logs to monitor. The list of logs is based
// on Google's signed list. Logs can also be added and removed manually.
package loglist
import (
"context"
"fmt"
"time"
"gitlab.torproject.org/rgdd/ct/pkg/metadata"
"rgdd.se/silentct/internal/ioutil"
)
type Config struct {
PermitBootstrap bool // Get and create initial log metadata if nothing valid is available on disk
MetadataFile string // Path to a dynamically updated metadata file that can be read/written
HistoryDirectory string // Existing directory to store the history of downloaded metadata
// Optional
MetadataInFuture time.Duration // How wrong the metadata timestamp is allowed to be wrt. future dating
MetadataGetsStale time.Duration // How long until the metadata is considered stale
MetadataIsRecent time.Duration // How long the metadata is considered recent
HTTPTimeout time.Duration // Timeout when fetching metadata
StaticLogs []metadata.Log // Takes precedence over the dynamically downloaded metadata
RemoveLogs []metadata.LogKey // Logs in the downloaded metadata with these keys are ignored
}
type LogList struct {
cfg Config
md metadata.Metadata
source metadata.Loader
}
func New(cfg Config) (LogList, error) {
if cfg.MetadataInFuture == 0 {
cfg.MetadataInFuture = 24 * time.Hour
}
if cfg.MetadataGetsStale == 0 {
cfg.MetadataGetsStale = 30 * 24 * time.Hour
}
if cfg.MetadataIsRecent == 0 {
cfg.MetadataIsRecent = 1 * time.Hour
}
if cfg.HTTPTimeout == 0 {
cfg.HTTPTimeout = 10 * time.Second
}
for i, log := range cfg.StaticLogs {
if err := checkLog(log); err != nil {
return LogList{}, fmt.Errorf("static logs: index %d: %v", i, err)
}
}
for i, key := range cfg.RemoveLogs {
if _, err := key.ID(); err != nil {
return LogList{}, fmt.Errorf("remove logs: index %d: %v", i, err)
}
}
s := metadata.NewHTTPSource(metadata.HTTPSourceOptions{Name: "google"})
ll := LogList{cfg: cfg, source: &s}
if err := ioutil.DirectoriesExist([]string{cfg.HistoryDirectory}); err != nil {
return LogList{}, err
}
if err := ioutil.ReadJSON(cfg.MetadataFile, &ll.md); err != nil {
if !ll.cfg.PermitBootstrap {
return LogList{}, err
}
if _, _, err := ll.Update(context.Background()); err != nil {
return LogList{}, err
}
}
return ll, nil
}
func (ll *LogList) IsRecent() bool {
return time.Now().Before(ll.md.CreatedAt.Add(ll.cfg.MetadataIsRecent))
}
func (ll *LogList) IsStale() bool {
return time.Now().After(ll.md.CreatedAt.Add(ll.cfg.MetadataGetsStale))
}
func (ll *LogList) Generate() []metadata.Log {
var configure []metadata.Log
for _, log := range ll.cfg.StaticLogs {
configure = append(configure, log)
}
for _, operator := range ll.md.Operators {
for _, log := range operator.Logs {
if findKey(ll.cfg.RemoveLogs, log) {
continue // static configuration says to remove this log
}
if findLog(configure, log) {
continue // static configuration takes precedence
}
if skipLog(log) {
continue // not in a state where it makes sense to monitor
}
configure = append(configure, log)
}
}
return configure
}
func (ll *LogList) Update(ctx context.Context) (added []metadata.Log, removed []metadata.Log, err error) {
b, md, err := ll.archiveDynamic(ctx)
if err != nil {
return
}
if err = ioutil.CommitData(ll.cfg.MetadataFile, b); err != nil {
return
}
added, removed = metadataLogDiff(ll.md, md)
ll.md = md
return
}
func (ll *LogList) archiveDynamic(ctx context.Context) ([]byte, metadata.Metadata, error) {
llctx, cancel := context.WithTimeout(ctx, ll.cfg.HTTPTimeout)
defer cancel()
msg, sig, md, err := ll.source.Load(llctx)
if err != nil {
return nil, metadata.Metadata{}, err
}
if future := time.Now().Add(ll.cfg.MetadataInFuture); md.CreatedAt.After(future) {
return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is in the future", md.CreatedAt)
}
if md.CreatedAt.Before(ll.md.CreatedAt) {
return nil, metadata.Metadata{}, fmt.Errorf("list created at %v is older than the current list", md.CreatedAt)
}
// FIXME: consider only archiving on major version bumps
path := fmt.Sprintf("%s/%s.json", ll.cfg.HistoryDirectory, time.Now().Format("2006-01-02_1504"))
if err := ioutil.CommitData(path, msg); err != nil {
return nil, metadata.Metadata{}, err
}
if err := ioutil.CommitData(path+".sig", sig); err != nil {
return nil, metadata.Metadata{}, err
}
return msg, md, nil
}
|
