aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--scripts/example.lst5
-rwxr-xr-xscripts/test.sh113
2 files changed, 118 insertions, 0 deletions
diff --git a/scripts/example.lst b/scripts/example.lst
new file mode 100644
index 0000000..a3ccfbf
--- /dev/null
+++ b/scripts/example.lst
@@ -0,0 +1,5 @@
+*.pulls.name
+www.rgdd.se
+www.torproject.org
+xyz.not.a.domain.that.resolves.example.org
+not a domain
diff --git a/scripts/test.sh b/scripts/test.sh
new file mode 100755
index 0000000..beac64f
--- /dev/null
+++ b/scripts/test.sh
@@ -0,0 +1,113 @@
+#!/bin/bash
+
+#
+# A script that debugs how different onion-grab configurations perform with
+# regards to successful connections, connection errors, found onions, etc. To
+# conduct these tests from multiple vantage points, Mullvad VPN is used.
+#
+
+###
+# Options you will likely want to tune yourself
+###
+relays=(\
+ se-sto-wg-001\
+ us-nyc-wg-501\
+ au-mel-wg-301\
+)
+limits=(\
+ 5\
+ 10\
+)
+num_workers=16
+input_file=example.lst
+
+###
+# Other options
+###
+tmp_dir="$(mktemp -d)"
+out_dir="data/$(date +%Y%m%d-%H%M%S)"
+mullvad_wait_s=5
+timeout=30s
+
+set -eu
+function main() {
+ num_sites="$(wc -l $input_file | cut -d' ' -f1)"
+ debug "relays=${#relays[@]}"
+ debug "limits=${#limits[@]}"
+ debug "numWorkers=$num_workers"
+ debug "inputFile=$input_file ($num_sites sites)"
+ debug "tmp_dir=$tmp_dir"
+ debug "out_dir=$out_dir"
+ debug "mullvad_wait_s=$mullvad_wait_s"
+
+ num_test="$(( ${#limits[@]} * ${#relays[@]} ))"
+ debug "about to start $num_test tests"
+
+ runtime_s=0
+ for limit in ${limits[@]}; do
+ for relay in ${relays[@]}; do
+ seconds=$(( $num_sites / $limit ))
+ runtime_s=$(( $runtime_s + $seconds ))
+ done
+ done
+ now=$(date +%s)
+ estimated_done=$(( $now + $runtime_s ))
+ debug "estimated done? earliest $(date -d @$estimated_done +"%Y-%m-%d %H:%M:%S %Z")"
+
+ i=1
+ for limit in ${limits[@]}; do
+ for relay in ${relays[@]}; do
+ mullvad disconnect >/dev/null
+ mullvad relay set hostname $relay >/dev/null
+ mullvad connect >/dev/null
+ sleep $mullvad_wait_s
+
+ ip=$(curl -s https://am.i.mullvad.net)
+ debug "working on test $i/$num_test at relay $relay ($ip) with limit $limit sites/s"
+
+ stdout_file=$tmp_dir/$relay-$limit.stdout
+ stderr_file=$tmp_dir/$relay-$limit.stderr
+ onion-grab -i $input_file -t $timeout -l $limit -w $num_workers >$stdout_file 2>$stderr_file
+
+ i=$(( $i + 1 ))
+ done
+ done
+
+ mkdir -p $out_dir
+ for limit in ${limits[@]}; do
+ dup_file=$tmp_dir/dup.txt
+
+ rm -f $dup_file
+ for relay in ${relays[@]}; do
+ stdout_file=$tmp_dir/$relay-$limit.stdout
+ stderr_file=$tmp_dir/$relay-$limit.stderr
+ cut -d' ' -f1 $stdout_file >> $dup_file
+
+ num_onion=$(wc -l $stdout_file | cut -d' ' -f1)
+ processed=$(grep Processed $stderr_file | tail -n1 | xargs | cut -d' ' -f2)
+ success=$(grep Success $stderr_file | tail -n1 | xargs | cut -d' ' -f2)
+ failure=$(grep Failure $stderr_file | tail -n1 | xargs | cut -d' ' -f2)
+ failure_dns=$(grep DNS $stderr_file | tail -n1 | xargs | cut -d' ' -f2)
+ failure_dns_detailed=$(grep DNS $stderr_file | tail -n1 | xargs | cut -d' ' -f3-)
+ info "relay:$relay limit:$limit onions:$num_onion connected:$success/$processed dns-error:$failure_dns/$failure $failure_dns_detailed"
+
+ output_file="$out_dir/$relay-l$limit.txt"
+ mv $stdout_file $output_file
+ done
+
+ num_sites=$(sort -Vu $dup_file | wc -l | cut -d' ' -f1)
+ info "all relays with limit $limit sites/s: $num_sites unique sites with onion locations found"
+ done
+
+ info "stored collected test data in $out_dir"
+ rm -rf $tmp_dir
+}
+
+function debug() {
+ echo "$(date +"%Y-%m-%d %H:%M:%S %Z") [DEBU] $@" >&2
+}
+function info() {
+ echo "$(date +"%Y-%m-%d %H:%M:%S %Z") [INFO] $@" >&2
+}
+
+main $@