diff options
Diffstat (limited to 'main.go')
-rw-r--r-- | main.go | 19 |
1 files changed, 18 insertions, 1 deletions
@@ -217,6 +217,7 @@ func workGenerator(ctx context.Context, opts options.Options, fp *os.File, quest buf := make([]byte, 0, opts.MaxFileBuffer*1024*1024) scanner.Buffer(buf, opts.MaxFileBuffer*1024*1024) + // roll-up to the requested start line nextLine := int64(0) if opts.StartLineInclusive > nextLine { for scanner.Scan() { @@ -233,17 +234,32 @@ func workGenerator(ctx context.Context, opts options.Options, fp *os.File, quest } } + // initialize ticker ticker := time.NewTicker(opts.MetricsInterval) defer ticker.Stop() - startTime := time.Now().Unix() latestTime := startTime latestCount := opts.StartLineInclusive + + // initialize rate-limit + limit := time.NewTicker(time.Second) + defer limit.Stop() + readCount := 0 + for scanner.Scan() { if opts.EndLineExclusive > 0 && nextLine == opts.EndLineExclusive { break } + if readCount == opts.Limit { + select { + case <-ctx.Done(): + return nextLine, false + case <-limit.C: + readCount = 0 + } + } + select { case <-ctx.Done(): return nextLine, false @@ -259,6 +275,7 @@ func workGenerator(ctx context.Context, opts options.Options, fp *os.File, quest latestTime = now case questionCh <- qna.Question{Domain: scanner.Text()}: nextLine++ + readCount++ } } |