diff options
Diffstat (limited to 'collect.go')
-rw-r--r-- | collect.go | 15 |
1 files changed, 13 insertions, 2 deletions
@@ -47,13 +47,21 @@ func collect(opts options) error { sigs := make(chan os.Signal, 1) defer close(sigs) + // Sometimes some worker in scanner.Fetcher isn't shutdown + // properly despite the parent context (including getRanges) + // being done. The below is an ugly hack to avoid hanging. + wait := time.Second * 5 // TODO: set higher with real runs signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) select { case <-sigs: - fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait...\n") + fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait %v...\n", wait) cancel() case <-ctx.Done(): } + select { + case <-time.After(wait): + os.Exit(0) + } }() for _, log := range utils.Logs(md) { @@ -77,7 +85,10 @@ func collect(opts options) error { cancel() return } - cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{UserAgent: "wip2"}) + cli, err := client.New(string(log.URL), + &http.Client{Transport: &http.Transport{IdleConnTimeout: 120 * time.Second}}, + jsonclient.Options{UserAgent: opts.httpAgent}, + ) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: %s: %v\n", *log.Description, err) cancel() |