diff options
author | Rasmus Dahlberg <rasmus@rgdd.se> | 2023-03-23 11:09:54 +0100 |
---|---|---|
committer | Rasmus Dahlberg <rasmus@rgdd.se> | 2023-03-23 11:58:16 +0100 |
commit | ad9fb49670e28414637761bac4b8e8940e2d6770 (patch) | |
tree | 920b0d7a266dbeed0ff0aa442752adb60d585748 /cmd_collect.go | |
parent | 38df474cf30b0b1d077c8d53b353a859af99c7d6 (diff) |
Automate handling of notice file
Here's a hacky tool to migrate our ongoing v0.0.1 measurement once it's
done. I.e., just split-up the NOTICE prints we have in collect.stdout,
putting them in per-log notice files that happens automatically now.
```
// Package main provides a hacky tool that extracts NOTICE: <log desc> prints
// from a file collect.stdout, putting them in the logs data directories as
// notice.txt. Only meant to migrate away from v0.0.1 that did not store
// per-log notice files automatically, which makes things less error-prone.
package main
import (
"bytes"
"encoding/json"
"fmt"
logger "log"
"os"
"strings"
"gitlab.torproject.org/rgdd/ct/pkg/metadata"
)
func main() {
directory := "../data"
logDirectory := fmt.Sprintf("%s/logs", directory)
noticeFile := "../collect.stdout"
b, err := os.ReadFile(fmt.Sprintf("%s/metadata.json", directory))
if err != nil {
logger.Fatal(err)
}
var md metadata.Metadata
if err := json.Unmarshal(b, &md); err != nil {
logger.Fatal(err)
}
if b, err = os.ReadFile(noticeFile); err != nil {
logger.Fatal(err)
}
lines := bytes.Split(b, []byte("\n"))
for _, log := range logs(md) {
id, _ := log.Key.ID()
desc := *log.Description
var notes []byte
var numNotes int
for _, line := range lines[:len(lines)-1] {
if strings.Contains(string(line), fmt.Sprintf("NOTICE: %s", desc)) {
notes = append(notes, line...)
notes = append(notes, []byte("\n")...)
numNotes += 1
}
}
if len(notes) == 0 {
logger.Printf("%s: no notices", desc)
continue
}
logger.Printf("%s: %d notices", desc, numNotes)
if err := os.WriteFile(fmt.Sprintf("%s/%x/notice.txt", logDirectory, id[:]), notes, 0644); err != nil {
logger.Fatal(err)
}
}
}
func logs(md metadata.Metadata) (logs []metadata.Log) {
for _, operators := range md.Operators {
for _, log := range operators.Logs {
if log.Description == nil {
logger.Printf("WARNING: skipping log without description")
continue
}
if log.State == nil {
continue // skip logs with unknown states
}
if log.State.Name == metadata.LogStatePending {
continue // pending logs do not count towards CT-compliance
}
if log.State.Name == metadata.LogStateRetired {
continue // retired logs are not necessarily reachable
}
if log.State.Name == metadata.LogStateRejected {
continue // rejected logs do not count towards CT-compliance
}
logs = append(logs, log)
}
}
return
}
```
Diffstat (limited to 'cmd_collect.go')
-rw-r--r-- | cmd_collect.go | 29 |
1 files changed, 23 insertions, 6 deletions
diff --git a/cmd_collect.go b/cmd_collect.go index 4d93271..742884a 100644 --- a/cmd_collect.go +++ b/cmd_collect.go @@ -3,7 +3,6 @@ package main import ( "container/heap" "context" - "crypto/sha256" "encoding/json" "fmt" logger "log" @@ -121,15 +120,18 @@ func collect(opts options) error { // chunk that a single sequencer can verify and persist // callback := func(eb scanner.EntryBatch) { - leafHashes := [][sha256.Size]byte{} + c := &chunk.Chunk{Start: eb.Start} for i := 0; i < len(eb.Entries); i++ { - leafHashes = append(leafHashes, merkle.HashLeafNode(eb.Entries[i].LeafInput)) + c.LeafHashes = append(c.LeafHashes, merkle.HashLeafNode(eb.Entries[i].LeafInput)) } - sans, errs := x509.SANsFromLeafEntries(eb.Start, eb.Entries) + + var errs []error + c.SANs, errs = x509.SANsFromLeafEntries(eb.Start, eb.Entries) for _, err := range errs { - logger.Printf("NOTICE: %s: %v", *log.Description, err) + c.Notes = append(c.Notes, fmt.Sprintf("NOTICE: %s: %v", *log.Description, err)) } - chunksCh <- &chunk.Chunk{eb.Start, leafHashes, sans} + + chunksCh <- c } if err := fetcher.Run(ctx, callback); err != nil { @@ -275,6 +277,21 @@ func persist(c *chunk.Chunk, return false, err } + // Persist notes to disk + if len(c.Notes) > 0 { + fpn, err := os.OpenFile(fmt.Sprintf("%s/%x/%s", opts.logDirectory, logID, opts.noticeFile), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return false, err + } + defer fpn.Close() + if _, err := fpn.WriteString(strings.Join(c.Notes, "\n") + "\n"); err != nil { + return false, err + } + if err := fpn.Sync(); err != nil { + return false, err + } + } + // Persist new tree state to disk b, err := json.Marshal(&newTH) if err != nil { |