aboutsummaryrefslogtreecommitdiff
path: root/main.go
diff options
context:
space:
mode:
Diffstat (limited to 'main.go')
-rw-r--r--main.go99
1 files changed, 43 insertions, 56 deletions
diff --git a/main.go b/main.go
index a68da1e..0efac30 100644
--- a/main.go
+++ b/main.go
@@ -3,8 +3,8 @@ package main
import (
"bufio"
"context"
+ "fmt"
"log"
- logger "log"
"net/http"
"os"
"os/signal"
@@ -14,41 +14,28 @@ import (
"git.cs.kau.se/rasmoste/find-onion/internal/onionloc"
"git.cs.kau.se/rasmoste/find-onion/internal/options"
+ "git.cs.kau.se/rasmoste/find-onion/internal/qna"
)
-type match struct {
- dnsName string
- onionName string
- httpHeader bool
- htmlAttr bool
-}
-
func main() {
opts := options.Parse()
cli := &http.Client{
Transport: &http.Transport{
- DisableKeepAlives: true,
+ DisableKeepAlives: true,
+ MaxResponseHeaderBytes: opts.MaxResponse,
},
}
- // sanCh is a channel of names that our workers should visit
- sanCh := make(chan string, 2*opts.NumWorkers)
- defer close(sanCh)
-
- // matchCh is a channel of matches, i.e., sites that had Onion-Location
- matchCh := make(chan match, 2*opts.NumWorkers)
- defer close(matchCh)
+ questionCh := make(chan qna.Question, 2*opts.NumWorkers)
+ defer close(questionCh)
- // missCh is a channel of misses, i.e., the number of sites that had not
- // configured Onion-Location. True means HTTP GET succeeded.
- missCh := make(chan bool, 2*opts.NumWorkers)
- defer close(missCh)
+ answerCh := make(chan qna.Answer, 2*opts.NumWorkers)
+ defer close(answerCh)
var wg sync.WaitGroup
defer wg.Wait()
ctx, cancel := context.WithCancel(context.Background())
- defer log.Printf("INFO: completed")
log.Printf("INFO: starting await handler, ctrl+C to exit\n")
go func() {
wg.Add(1)
@@ -61,37 +48,37 @@ func main() {
go func() {
wg.Add(1)
defer wg.Done()
- workHandler(ctx, opts, cli, sanCh, matchCh, missCh)
+ workHandler(ctx, opts, cli, questionCh, answerCh)
}()
}
- log.Printf("INFO: starting work receiver\n")
+ log.Printf("INFO: starting work aggregator\n")
go func() {
wg.Add(1)
defer wg.Done()
- workAggregator(ctx, cancel, opts, matchCh, missCh)
+ workAggregator(ctx, cancel, opts, answerCh)
}()
log.Printf("INFO: generating work\n")
- workGenerator(ctx, cancel, opts, sanCh)
+ workGenerator(ctx, cancel, opts, questionCh)
time.Sleep(time.Second)
defer cancel()
+ defer time.Sleep(2 * opts.Timeout)
+ defer log.Printf("INFO: about to exit in %v", 2*opts.Timeout)
for {
select {
case <-ctx.Done():
- logger.Printf("DEBUG: context cancelled, exiting...\n")
+ log.Printf("INFO: context cancelled")
return
- case <-time.After(1 * time.Second):
+ case <-time.After(time.Second):
}
- numMessages := len(sanCh) + len(matchCh) + len(missCh)
+ numMessages := len(questionCh) + len(answerCh)
if numMessages == 0 {
- logger.Printf("Exit scheduled in %v from now", 2*opts.Timeout)
- time.Sleep(2 * opts.Timeout)
- return // TODO: fix a cleaner exit
+ return
}
- logger.Printf("DEBUG: waiting for %d messages to be processed before exit", numMessages)
+ log.Printf("DEBUG: waiting for %d messages to be processed before exit", numMessages)
}
}
@@ -107,47 +94,45 @@ func await(ctx context.Context, cancel context.CancelFunc) {
}
}
-func workHandler(ctx context.Context, opts options.Options, cli *http.Client,
- sanCh chan string, matchCh chan match, missCh chan bool) {
+func workHandler(ctx context.Context, opts options.Options, cli *http.Client, questionCh chan qna.Question, answerCh chan qna.Answer) {
for {
select {
case <-ctx.Done():
return
- case san := <-sanCh:
+ case question := <-questionCh:
func() {
cctx, cancel := context.WithTimeout(ctx, opts.Timeout)
defer cancel()
- req, err := http.NewRequestWithContext(cctx, http.MethodGet, "https://"+san, nil)
+ req, err := http.NewRequestWithContext(cctx, http.MethodGet, "https://"+question.Domain, nil)
if err != nil {
- logger.Printf("DEBUG: %s: %v", san, err)
return
}
rsp, err := cli.Do(req)
if err != nil {
- missCh <- false
+ answerCh <- qna.Answer{question.Domain, false, false, false, ""}
return
}
defer rsp.Body.Close()
v, ok := onionloc.HTTP(rsp)
if ok {
- matchCh <- match{dnsName: san, onionName: v, httpHeader: true}
+ answerCh <- qna.Answer{question.Domain, true, true, false, v}
return
}
v, ok = onionloc.HTML(rsp)
if ok {
- matchCh <- match{dnsName: san, onionName: v, htmlAttr: true}
+ answerCh <- qna.Answer{question.Domain, true, false, true, v}
return
}
- missCh <- true
+ answerCh <- qna.Answer{question.Domain, true, false, false, ""}
}()
}
}
}
-func workAggregator(ctx context.Context, _ context.CancelFunc, opts options.Options, matchCh chan match, missCh chan bool) {
+func workAggregator(ctx context.Context, _ context.CancelFunc, opts options.Options, answerCh chan qna.Answer) {
ticker := time.NewTicker(opts.MetricsInterval)
defer ticker.Stop()
@@ -155,7 +140,7 @@ func workAggregator(ctx context.Context, _ context.CancelFunc, opts options.Opti
numOnionLocation := 0
numVisits := 0
output := func() {
- logger.Printf("INFO: %d/%d connected, %d matched\n", numConnected, numVisits, numOnionLocation)
+ log.Printf("INFO: %d/%d connected, %d matched\n", numConnected, numVisits, numOnionLocation)
}
defer output()
@@ -163,16 +148,16 @@ func workAggregator(ctx context.Context, _ context.CancelFunc, opts options.Opti
select {
case <-ctx.Done():
return
- case m := <-matchCh:
+ case a := <-answerCh:
numVisits += 1
- numConnected += 1
- numOnionLocation += 1
+ if !a.OK {
+ continue
+ }
- log.Printf("INFO: found match %s (HTTP:%v HTML:%v)\n", m.dnsName, m.httpHeader, m.htmlAttr)
- case connected := <-missCh:
- numVisits += 1
- if connected {
- numConnected += 1
+ numConnected += 1
+ if a.HTTP || a.HTML {
+ numOnionLocation += 1
+ fmt.Printf("http:%v html:%v domain:%s onion:%s \n", a.HTTP, a.HTML, a.Domain, a.Onion)
}
case <-ticker.C:
output()
@@ -180,20 +165,22 @@ func workAggregator(ctx context.Context, _ context.CancelFunc, opts options.Opti
}
}
-func workGenerator(ctx context.Context, cancel context.CancelFunc, opts options.Options, sanCh chan string) {
+func workGenerator(ctx context.Context, cancel context.CancelFunc, opts options.Options, questionCh chan qna.Question) {
fp, err := os.OpenFile(opts.InputFile, os.O_RDONLY, 0644)
if err != nil {
- logger.Printf("ERROR: %v", err)
+ log.Printf("ERROR: %v", err)
cancel()
return
}
defer fp.Close()
scanner := bufio.NewScanner(fp)
- max := 1024 * 1024
+ max := 2 * 256 * opts.NumWorkers
buf := make([]byte, 0, max)
scanner.Buffer(buf, max)
+ // TODO: track which line we would have to start from to be sure that
+ // we're not missing any domains on ctrl+C, OK if we go back too much?
for scanner.Scan() {
select {
case <-ctx.Done():
@@ -202,8 +189,8 @@ func workGenerator(ctx context.Context, cancel context.CancelFunc, opts options.
}
for {
- if len(sanCh) < cap(sanCh) {
- sanCh <- scanner.Text()
+ if len(questionCh) < cap(questionCh) {
+ questionCh <- qna.Question{Domain: scanner.Text()}
break
}
select {
generated by cgit v1.2.3 (git 2.39.1) at 2024-10-22 13:34:42 +0000