more wip collect

1 files changed, 13 insertions, 2 deletions

diff --git a/collect.go b/collect.go
index 739400f..1e469b3 100644
--- a/collect.go
+++ b/collect.go
@@ -47,13 +47,21 @@ func collect(opts options) error {
 		sigs := make(chan os.Signal, 1)
 		defer close(sigs)
 
+		// Sometimes some worker in scanner.Fetcher isn't shutdown
+		// properly despite the parent context (including getRanges)
+		// being done.  The below is an ugly hack to avoid hanging.
+		wait := time.Second * 5 // TODO: set higher with real runs
 		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 		select {
 		case <-sigs:
-			fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait...\n")
+			fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait %v...\n", wait)
 			cancel()
 		case <-ctx.Done():
 		}
+		select {
+		case <-time.After(wait):
+			os.Exit(0)
+		}
 	}()
 
 	for _, log := range utils.Logs(md) {
@@ -77,7 +85,10 @@ func collect(opts options) error {
 				cancel()
 				return
 			}
-			cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{UserAgent: "wip2"})
+			cli, err := client.New(string(log.URL),
+				&http.Client{Transport: &http.Transport{IdleConnTimeout: 120 * time.Second}},
+				jsonclient.Options{UserAgent: opts.httpAgent},
+			)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "ERROR: %s: %v\n", *log.Description, err)
 				cancel()