Add input file read rate-limit

1 files changed, 18 insertions, 1 deletions

diff --git a/main.go b/main.go
index 55e7989..072924a 100644
--- a/main.go
+++ b/main.go
@@ -217,6 +217,7 @@ func workGenerator(ctx context.Context, opts options.Options, fp *os.File, quest
 	buf := make([]byte, 0, opts.MaxFileBuffer*1024*1024)
 	scanner.Buffer(buf, opts.MaxFileBuffer*1024*1024)
 
+	// roll-up to the requested start line
 	nextLine := int64(0)
 	if opts.StartLineInclusive > nextLine {
 		for scanner.Scan() {
@@ -233,17 +234,32 @@ func workGenerator(ctx context.Context, opts options.Options, fp *os.File, quest
 		}
 	}
 
+	// initialize ticker
 	ticker := time.NewTicker(opts.MetricsInterval)
 	defer ticker.Stop()
-
 	startTime := time.Now().Unix()
 	latestTime := startTime
 	latestCount := opts.StartLineInclusive
+
+	// initialize rate-limit
+	limit := time.NewTicker(time.Second)
+	defer limit.Stop()
+	readCount := 0
+
 	for scanner.Scan() {
 		if opts.EndLineExclusive > 0 && nextLine == opts.EndLineExclusive {
 			break
 		}
 
+		if readCount == opts.Limit {
+			select {
+			case <-ctx.Done():
+				return nextLine, false
+			case <-limit.C:
+				readCount = 0
+			}
+		}
+
 		select {
 		case <-ctx.Done():
 			return nextLine, false
@@ -259,6 +275,7 @@ func workGenerator(ctx context.Context, opts options.Options, fp *os.File, quest
 			latestTime = now
 		case questionCh <- qna.Question{Domain: scanner.Text()}:
 			nextLine++
+			readCount++
 		}
 	}