diff options
| -rw-r--r-- | scripts/example.lst | 5 | ||||
| -rwxr-xr-x | scripts/test.sh | 113 |
2 files changed, 118 insertions, 0 deletions
diff --git a/scripts/example.lst b/scripts/example.lst new file mode 100644 index 0000000..a3ccfbf --- /dev/null +++ b/scripts/example.lst @@ -0,0 +1,5 @@ +*.pulls.name +www.rgdd.se +www.torproject.org +xyz.not.a.domain.that.resolves.example.org +not a domain diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..beac64f --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# +# A script that debugs how different onion-grab configurations perform with +# regards to successful connections, connection errors, found onions, etc. To +# conduct these tests from multiple vantage points, Mullvad VPN is used. +# + +### +# Options you will likely want to tune yourself +### +relays=(\ + se-sto-wg-001\ + us-nyc-wg-501\ + au-mel-wg-301\ +) +limits=(\ + 5\ + 10\ +) +num_workers=16 +input_file=example.lst + +### +# Other options +### +tmp_dir="$(mktemp -d)" +out_dir="data/$(date +%Y%m%d-%H%M%S)" +mullvad_wait_s=5 +timeout=30s + +set -eu +function main() { + num_sites="$(wc -l $input_file | cut -d' ' -f1)" + debug "relays=${#relays[@]}" + debug "limits=${#limits[@]}" + debug "numWorkers=$num_workers" + debug "inputFile=$input_file ($num_sites sites)" + debug "tmp_dir=$tmp_dir" + debug "out_dir=$out_dir" + debug "mullvad_wait_s=$mullvad_wait_s" + + num_test="$(( ${#limits[@]} * ${#relays[@]} ))" + debug "about to start $num_test tests" + + runtime_s=0 + for limit in ${limits[@]}; do + for relay in ${relays[@]}; do + seconds=$(( $num_sites / $limit )) + runtime_s=$(( $runtime_s + $seconds )) + done + done + now=$(date +%s) + estimated_done=$(( $now + $runtime_s )) + debug "estimated done? earliest $(date -d @$estimated_done +"%Y-%m-%d %H:%M:%S %Z")" + + i=1 + for limit in ${limits[@]}; do + for relay in ${relays[@]}; do + mullvad disconnect >/dev/null + mullvad relay set hostname $relay >/dev/null + mullvad connect >/dev/null + sleep $mullvad_wait_s + + ip=$(curl -s https://am.i.mullvad.net) + debug "working on test $i/$num_test at relay $relay ($ip) with limit $limit sites/s" + + stdout_file=$tmp_dir/$relay-$limit.stdout + stderr_file=$tmp_dir/$relay-$limit.stderr + onion-grab -i $input_file -t $timeout -l $limit -w $num_workers >$stdout_file 2>$stderr_file + + i=$(( $i + 1 )) + done + done + + mkdir -p $out_dir + for limit in ${limits[@]}; do + dup_file=$tmp_dir/dup.txt + + rm -f $dup_file + for relay in ${relays[@]}; do + stdout_file=$tmp_dir/$relay-$limit.stdout + stderr_file=$tmp_dir/$relay-$limit.stderr + cut -d' ' -f1 $stdout_file >> $dup_file + + num_onion=$(wc -l $stdout_file | cut -d' ' -f1) + processed=$(grep Processed $stderr_file | tail -n1 | xargs | cut -d' ' -f2) + success=$(grep Success $stderr_file | tail -n1 | xargs | cut -d' ' -f2) + failure=$(grep Failure $stderr_file | tail -n1 | xargs | cut -d' ' -f2) + failure_dns=$(grep DNS $stderr_file | tail -n1 | xargs | cut -d' ' -f2) + failure_dns_detailed=$(grep DNS $stderr_file | tail -n1 | xargs | cut -d' ' -f3-) + info "relay:$relay limit:$limit onions:$num_onion connected:$success/$processed dns-error:$failure_dns/$failure $failure_dns_detailed" + + output_file="$out_dir/$relay-l$limit.txt" + mv $stdout_file $output_file + done + + num_sites=$(sort -Vu $dup_file | wc -l | cut -d' ' -f1) + info "all relays with limit $limit sites/s: $num_sites unique sites with onion locations found" + done + + info "stored collected test data in $out_dir" + rm -rf $tmp_dir +} + +function debug() { + echo "$(date +"%Y-%m-%d %H:%M:%S %Z") [DEBU] $@" >&2 +} +function info() { + echo "$(date +"%Y-%m-%d %H:%M:%S %Z") [INFO] $@" >&2 +} + +main $@ |