aboutsummaryrefslogtreecommitdiff
Commit message (Collapse)AuthorAgeFilesLines
* Drop debug printRasmus Dahlberg2023-03-231-1/+0
|
* Automate handling of notice fileRasmus Dahlberg2023-03-233-28/+54
| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Here's a hacky tool to migrate our ongoing v0.0.1 measurement once it's done. I.e., just split-up the NOTICE prints we have in collect.stdout, putting them in per-log notice files that happens automatically now. ``` // Package main provides a hacky tool that extracts NOTICE: <log desc> prints // from a file collect.stdout, putting them in the logs data directories as // notice.txt. Only meant to migrate away from v0.0.1 that did not store // per-log notice files automatically, which makes things less error-prone. package main import ( "bytes" "encoding/json" "fmt" logger "log" "os" "strings" "gitlab.torproject.org/rgdd/ct/pkg/metadata" ) func main() { directory := "../data" logDirectory := fmt.Sprintf("%s/logs", directory) noticeFile := "../collect.stdout" b, err := os.ReadFile(fmt.Sprintf("%s/metadata.json", directory)) if err != nil { logger.Fatal(err) } var md metadata.Metadata if err := json.Unmarshal(b, &md); err != nil { logger.Fatal(err) } if b, err = os.ReadFile(noticeFile); err != nil { logger.Fatal(err) } lines := bytes.Split(b, []byte("\n")) for _, log := range logs(md) { id, _ := log.Key.ID() desc := *log.Description var notes []byte var numNotes int for _, line := range lines[:len(lines)-1] { if strings.Contains(string(line), fmt.Sprintf("NOTICE: %s", desc)) { notes = append(notes, line...) notes = append(notes, []byte("\n")...) numNotes += 1 } } if len(notes) == 0 { logger.Printf("%s: no notices", desc) continue } logger.Printf("%s: %d notices", desc, numNotes) if err := os.WriteFile(fmt.Sprintf("%s/%x/notice.txt", logDirectory, id[:]), notes, 0644); err != nil { logger.Fatal(err) } } } func logs(md metadata.Metadata) (logs []metadata.Log) { for _, operators := range md.Operators { for _, log := range operators.Logs { if log.Description == nil { logger.Printf("WARNING: skipping log without description") continue } if log.State == nil { continue // skip logs with unknown states } if log.State.Name == metadata.LogStatePending { continue // pending logs do not count towards CT-compliance } if log.State.Name == metadata.LogStateRetired { continue // retired logs are not necessarily reachable } if log.State.Name == metadata.LogStateRejected { continue // rejected logs do not count towards CT-compliance } logs = append(logs, log) } } return } ```
* Drop sanitize of SANsRasmus Dahlberg2023-03-233-179/+18
| | | | Less complex, we will just pass lines to Go's HTTP GET as is.
* Change default to /tmpRasmus Dahlberg2023-03-201-2/+2
|
* Add drafty assemble commandRasmus Dahlberg2023-03-203-6/+316
|
* Fork code snippets to sanitize DNS names (ascii)Rasmus Dahlberg2023-03-201-0/+85
|
* Smaller default paramsRasmus Dahlberg2023-03-181-4/+4
|
* Add notes from starting experimentRasmus Dahlberg2023-03-181-0/+43
|
* Manually add version+logID to STH structv0.0.1Rasmus Dahlberg2023-03-181-4/+2
| | | | Will probably make the data set manifest simpler.
* Light refactoringRasmus Dahlberg2023-03-181-5/+5
|
* Tweak default parametersRasmus Dahlberg2023-03-181-4/+4
| | | | Will put exactly how we run on our machine in README
* Print stats on shutdownRasmus Dahlberg2023-03-182-11/+16
|
* Fix few debug printsRasmus Dahlberg2023-03-182-4/+3
|
* Add one more max workersRasmus Dahlberg2023-03-181-0/+3
|
* Tune number of workersRasmus Dahlberg2023-03-184-5/+31
|
* remove temp fileRasmus Dahlberg2023-03-181-146/+0
|
* check that sans don't contain new linesRasmus Dahlberg2023-03-182-2/+153
|
* rename default dirRasmus Dahlberg2023-03-182-3/+3
|
* Move helpers to separate fileRasmus Dahlberg2023-03-182-38/+47
|
* Print metrics based on flag duration to stderrRasmus Dahlberg2023-03-184-10/+20
|
* update http headerRasmus Dahlberg2023-03-181-2/+2
|
* add creditRasmus Dahlberg2023-03-181-0/+3
|
* renaming files and moving aroundRasmus Dahlberg2023-03-185-33/+42
|
* more light refactoringRasmus Dahlberg2023-03-184-67/+56
|
* light refactoringRasmus Dahlberg2023-03-181-104/+104
|
* more wip collect, metricsRasmus Dahlberg2023-03-182-16/+106
|
* more wip collectRasmus Dahlberg2023-03-173-50/+68
|
* Make a few constants into flagsRasmus Dahlberg2023-03-172-26/+23
|
* Add uint64 to ctflagRasmus Dahlberg2023-03-171-0/+6
|
* more wip collectRasmus Dahlberg2023-03-172-3/+18
|
* more wip collectRasmus Dahlberg2023-03-172-19/+49
|
* Remove dead codeRasmus Dahlberg2023-03-171-9/+0
|
* Fork some snippets to get SANs parsingRasmus Dahlberg2023-03-171-0/+487
|
* wip collectRasmus Dahlberg2023-03-174-2/+412
|
* Add heap chunk and helpersRasmus Dahlberg2023-03-171-0/+86
|
* Add utilsRasmus Dahlberg2023-03-171-0/+56
|
* Add .gitignoreRasmus Dahlberg2023-03-171-0/+1
|
* Add drafty snapshot commandRasmus Dahlberg2023-03-174-3/+155
|
* Add drafty command skeletonRasmus Dahlberg2023-03-173-0/+111
|
* Copy internal ctflag packageRasmus Dahlberg2023-03-171-0/+84
|
* Add authors and licenceRasmus Dahlberg2023-03-172-0/+47
| | | | Add source URL once this has a more permanent home
* Fork merkle packageRasmus Dahlberg2023-03-172-0/+386
| | | | | To get ranged inclusion proofs that operate on leaf hashes rather than the leaf data. Should probably be fixed upstream at some point.
* Add go moduleRasmus Dahlberg2023-03-171-0/+3
|
* Add drafty sketch of ct-sans data collectionRasmus Dahlberg2023-03-171-0/+123
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-26 18:30:27 +0000