more wip collect

2 files changed, 18 insertions, 3 deletions

diff --git a/collect.go b/collect.go
index 739400f..1e469b3 100644
--- a/collect.go
+++ b/collect.go
@@ -47,13 +47,21 @@ func collect(opts options) error {
 		sigs := make(chan os.Signal, 1)
 		defer close(sigs)
 
+		// Sometimes some worker in scanner.Fetcher isn't shutdown
+		// properly despite the parent context (including getRanges)
+		// being done.  The below is an ugly hack to avoid hanging.
+		wait := time.Second * 5 // TODO: set higher with real runs
 		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 		select {
 		case <-sigs:
-			fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait...\n")
+			fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait %v...\n", wait)
 			cancel()
 		case <-ctx.Done():
 		}
+		select {
+		case <-time.After(wait):
+			os.Exit(0)
+		}
 	}()
 
 	for _, log := range utils.Logs(md) {
@@ -77,7 +85,10 @@ func collect(opts options) error {
 				cancel()
 				return
 			}
-			cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{UserAgent: "wip2"})
+			cli, err := client.New(string(log.URL),
+				&http.Client{Transport: &http.Transport{IdleConnTimeout: 120 * time.Second}},
+				jsonclient.Options{UserAgent: opts.httpAgent},
+			)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "ERROR: %s: %v\n", *log.Description, err)
 				cancel()
diff --git a/main.go b/main.go
index dea0df7..53cfd65 100644
--- a/main.go
+++ b/main.go
@@ -2,7 +2,7 @@
 //
 // Install:
 //
-//	go install gitlab.torproject.org/rgdd/ct/cmd/ct-sans@latest
+//	go install git.cs.kau.se/rasmoste/ct-sans@latest
 //
 // Usage:
 //
@@ -55,6 +55,8 @@ type options struct {
 	workersPerLog uint64
 	batchSize     uint64
 	persistSize   uint64
+
+	httpAgent string
 }
 
 func main() {
@@ -93,6 +95,8 @@ func main() {
 	opts.batchSize = 10
 	opts.persistSize = 1
 
+	opts.httpAgent = "wip"
+
 	// Hand-over to the respective subcommands
 	var err error
 	switch cmd := os.Args[1]; cmd {