diff options
| -rw-r--r-- | collect.go | 8 | ||||
| -rw-r--r-- | main.go | 41 |
2 files changed, 23 insertions, 26 deletions
@@ -87,7 +87,7 @@ func collect(opts options) error { } cli, err := client.New(string(log.URL), &http.Client{Transport: &http.Transport{IdleConnTimeout: 120 * time.Second}}, - jsonclient.Options{UserAgent: opts.httpAgent}, + jsonclient.Options{UserAgent: opts.HTTPAgent}, ) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: %s: %v\n", *log.Description, err) @@ -95,10 +95,10 @@ func collect(opts options) error { return } fetcher := scanner.NewFetcher(cli, &scanner.FetcherOptions{ - BatchSize: int(opts.batchSize), + BatchSize: int(opts.BatchSize), StartIndex: th.TreeSize, EndIndex: int64(sth.TreeSize), - ParallelFetch: int(opts.workersPerLog), + ParallelFetch: int(opts.WorkersPerLog), }) // @@ -150,7 +150,7 @@ func collect(opts options) error { } c = h.TPop() - putBack, err := persistChunk(cli, opts, id[:], int64(opts.persistSize), c) + putBack, err := persistChunk(cli, opts, id[:], int64(opts.PersistSize), c) if err != nil { cancel() fmt.Fprintf(os.Stderr, "ERROR: %s: %v\n", *log.Description, err) @@ -21,13 +21,13 @@ const usage = `ct-sans collects SANs in CT-logged certificates Usage: - ct-sans snapshot [-d DIRECTORY] + ct-sans snapshot [Options...] Refresh log lists, signed tree heads, and timestamps - ct-sans collect [-d DIRECTORY] + ct-sans collect [Options...] Collect SANs with regards to the current snapshot - ct-sans assemble [-d DIRECTORY] + ct-sans assemble [Options...] Assemble a dataset manifest and print a command that combines, sorts, and removes duplicate SANs that were collected. @@ -37,12 +37,20 @@ Help: Options: - -d, --directory: The ct-sans working directory (Default: "ct-sans") + -d, --directory: The ct-sans working directory (Default: "ct-sans") + -w, --workers: Number of parallel download workers per log (Default: 2) + -k, --batch-disk: Certificate batch size before persisting (Default: 16384) + -q, --batch-req: Certificate batch size to use in request (Default: 512) + -a, --http-agent: HTTP agent to use in all request (Default: "rgdd.se/ct-sans") ` type options struct { - Directory string + Directory string + WorkersPerLog uint64 + PersistSize uint64 + BatchSize uint64 + HTTPAgent string logDirectory string metadataFile string @@ -51,12 +59,6 @@ type options struct { sthFile string stateFile string sansFile string - - workersPerLog uint64 - batchSize uint64 - persistSize uint64 - - httpAgent string } func main() { @@ -69,8 +71,12 @@ func main() { fs := ctflag.NewFlagSet() opts := options{} ctflag.String(&fs, &opts.Directory, "directory", "d", "ct-sans") + ctflag.Uint64(&fs, &opts.WorkersPerLog, "workers", "w", 2) + ctflag.Uint64(&fs, &opts.PersistSize, "batch-disk", "k", 16384) + ctflag.Uint64(&fs, &opts.BatchSize, "batch-req", "q", 512) + ctflag.String(&fs, &opts.HTTPAgent, "http-agent", "a", "rgdd.se/ct-sans") - // Parse command-line options and hardcode default values + // Parse command-line options and hardcode additional values if err := ctflag.Parse(fs, os.Args[2:]); err != nil { if err == flag.ErrHelp { fmt.Fprintf(os.Stderr, usage) @@ -83,20 +89,11 @@ func main() { opts.logDirectory = opts.Directory + "/" + "logs" opts.metadataFile = "metadata.json" opts.metadataSignatureFile = "metadata.sig" - opts.metadataTimestampFile = "metadata.sig" + opts.metadataTimestampFile = "metadata.timestamp" opts.sthFile = "sth.json" opts.stateFile = "th.json" opts.sansFile = "sans.lst" - //opts.workersPerLog = 100 - //opts.batchSize = 100 - //opts.persistSize = 10000 - opts.workersPerLog = 1 - opts.batchSize = 10 - opts.persistSize = 1 - - opts.httpAgent = "wip" - // Hand-over to the respective subcommands var err error switch cmd := os.Args[1]; cmd { |