diff options
| -rw-r--r-- | collect.go | 15 | ||||
| -rw-r--r-- | main.go | 6 |
2 files changed, 18 insertions, 3 deletions
@@ -47,13 +47,21 @@ func collect(opts options) error { sigs := make(chan os.Signal, 1) defer close(sigs) + // Sometimes some worker in scanner.Fetcher isn't shutdown + // properly despite the parent context (including getRanges) + // being done. The below is an ugly hack to avoid hanging. + wait := time.Second * 5 // TODO: set higher with real runs signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) select { case <-sigs: - fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait...\n") + fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait %v...\n", wait) cancel() case <-ctx.Done(): } + select { + case <-time.After(wait): + os.Exit(0) + } }() for _, log := range utils.Logs(md) { @@ -77,7 +85,10 @@ func collect(opts options) error { cancel() return } - cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{UserAgent: "wip2"}) + cli, err := client.New(string(log.URL), + &http.Client{Transport: &http.Transport{IdleConnTimeout: 120 * time.Second}}, + jsonclient.Options{UserAgent: opts.httpAgent}, + ) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: %s: %v\n", *log.Description, err) cancel() @@ -2,7 +2,7 @@ // // Install: // -// go install gitlab.torproject.org/rgdd/ct/cmd/ct-sans@latest +// go install git.cs.kau.se/rasmoste/ct-sans@latest // // Usage: // @@ -55,6 +55,8 @@ type options struct { workersPerLog uint64 batchSize uint64 persistSize uint64 + + httpAgent string } func main() { @@ -93,6 +95,8 @@ func main() { opts.batchSize = 10 opts.persistSize = 1 + opts.httpAgent = "wip" + // Hand-over to the respective subcommands var err error switch cmd := os.Args[1]; cmd { |