aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--collect.go15
-rw-r--r--main.go6
2 files changed, 18 insertions, 3 deletions
diff --git a/collect.go b/collect.go
index 739400f..1e469b3 100644
--- a/collect.go
+++ b/collect.go
@@ -47,13 +47,21 @@ func collect(opts options) error {
sigs := make(chan os.Signal, 1)
defer close(sigs)
+ // Sometimes some worker in scanner.Fetcher isn't shutdown
+ // properly despite the parent context (including getRanges)
+ // being done. The below is an ugly hack to avoid hanging.
+ wait := time.Second * 5 // TODO: set higher with real runs
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
select {
case <-sigs:
- fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait...\n")
+ fmt.Fprintf(os.Stderr, "INFO: received shutdown signal, please wait %v...\n", wait)
cancel()
case <-ctx.Done():
}
+ select {
+ case <-time.After(wait):
+ os.Exit(0)
+ }
}()
for _, log := range utils.Logs(md) {
@@ -77,7 +85,10 @@ func collect(opts options) error {
cancel()
return
}
- cli, err := client.New(string(log.URL), &http.Client{}, jsonclient.Options{UserAgent: "wip2"})
+ cli, err := client.New(string(log.URL),
+ &http.Client{Transport: &http.Transport{IdleConnTimeout: 120 * time.Second}},
+ jsonclient.Options{UserAgent: opts.httpAgent},
+ )
if err != nil {
fmt.Fprintf(os.Stderr, "ERROR: %s: %v\n", *log.Description, err)
cancel()
diff --git a/main.go b/main.go
index dea0df7..53cfd65 100644
--- a/main.go
+++ b/main.go
@@ -2,7 +2,7 @@
//
// Install:
//
-// go install gitlab.torproject.org/rgdd/ct/cmd/ct-sans@latest
+// go install git.cs.kau.se/rasmoste/ct-sans@latest
//
// Usage:
//
@@ -55,6 +55,8 @@ type options struct {
workersPerLog uint64
batchSize uint64
persistSize uint64
+
+ httpAgent string
}
func main() {
@@ -93,6 +95,8 @@ func main() {
opts.batchSize = 10
opts.persistSize = 1
+ opts.httpAgent = "wip"
+
// Hand-over to the respective subcommands
var err error
switch cmd := os.Args[1]; cmd {