From 2bee2104c84628a68ef7124a1beefa4e1f98369e Mon Sep 17 00:00:00 2001
From: Rasmus Dahlberg <rasmus@rgdd.se>
Date: Mon, 20 Mar 2023 19:52:21 +0100
Subject: Add drafty assemble command

---
 main.go | 38 ++++++++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

(limited to 'main.go')

diff --git a/main.go b/main.go
index 3d9b3a7..88d6b90 100644
--- a/main.go
+++ b/main.go
@@ -30,14 +30,17 @@ Usage:
     Collect SANs with regards to the current snapshot
 
   ct-sans assemble [Options...]
-    Assemble a dataset manifest and print a command that combines,
-    sorts, and removes duplicate SANs that were collected.
+    Assemble dataset from the collected data (requires GNU sort)
 
 Help:
 
   ct-sans [-h] [--help]
 
-Options:
+Snapshot options:
+
+  -d, --directory:   The ct-sans working directory (Default: "data")
+
+Collect options:
 
   -d, --directory:   The ct-sans working directory (Default: "data")
   -w, --workers:     Max number of parallel download workers per log (Default: 2).
@@ -46,23 +49,42 @@ Options:
   -a, --http-agent:  HTTP agent to use in all request (Default: "git.cs.kau.se/rasmoste/ct-sans")
   -m, --metrics:     How often to emit metrics to stderr (Default: 16s)
 
+Asemble options:
+
+  -d, --directory:   The ct-sans working directory (Default: "data")
+  -b, --buffer-size: Max memory to use in Gigabytes (Default: 1)
+  -t, --temp-dir:    Temporary on-disk storage directory (Default: /tmp/ct-sans)
+  -p, --parallel:    Number of CPUs to use (Default: 1)
+
 `
 
 type options struct {
-	Directory       string
+	// Common options
+	Directory string
+
+	// Collect options
 	WorkersPerLog   uint64
 	PersistSize     uint64
 	BatchSize       uint64
 	HTTPAgent       string
 	MetricsInterval time.Duration
 
+	// Assemble options
+	BufferSize uint64
+	TempDir    string
+	Parallel   uint64
+
+	// Constants
 	logDirectory          string
+	archiveDirectory      string
 	metadataFile          string
 	metadataSignatureFile string
 	metadataTimestampFile string
 	sthFile               string
 	stateFile             string
 	sansFile              string
+	visitFile             string
+	noticeFile            string
 }
 
 func main() {
@@ -77,12 +99,17 @@ func main() {
 	fs := ctflag.NewFlagSet()
 	opts := options{}
 	ctflag.String(&fs, &opts.Directory, "directory", "d", "data")
+
 	ctflag.Uint64(&fs, &opts.WorkersPerLog, "workers", "w", 2)
 	ctflag.Uint64(&fs, &opts.PersistSize, "batch-disk", "k", 256)
 	ctflag.Uint64(&fs, &opts.BatchSize, "batch-req", "q", 128)
 	ctflag.String(&fs, &opts.HTTPAgent, "http-agent", "a", "git.cs.kau.se/rasmoste/ct-sans")
 	ctflag.Duration(&fs, &opts.MetricsInterval, "metrics", "m", 16*time.Second)
 
+	ctflag.Uint64(&fs, &opts.BufferSize, "buffer-size", "b", 1)
+	ctflag.String(&fs, &opts.TempDir, "temp-dir", "t", "/tmp/ct-sans")
+	ctflag.Uint64(&fs, &opts.Parallel, "parallel", "p", 1)
+
 	// Parse command-line options and hardcode additional values
 	if err := ctflag.Parse(fs, os.Args[2:]); err != nil {
 		if err == flag.ErrHelp {
@@ -94,12 +121,15 @@ func main() {
 		os.Exit(1)
 	}
 	opts.logDirectory = opts.Directory + "/" + "logs"
+	opts.archiveDirectory = opts.Directory + "/" + "archive"
 	opts.metadataFile = "metadata.json"
 	opts.metadataSignatureFile = "metadata.sig"
 	opts.metadataTimestampFile = "metadata.timestamp"
 	opts.sthFile = "sth.json"
 	opts.stateFile = "th.json"
 	opts.sansFile = "sans.lst"
+	opts.visitFile = "visit.lst"
+	opts.noticeFile = "notice.txt"
 
 	// Hand-over to the respective subcommands
 	var err error
-- 
cgit v1.2.3