|
|
|
|
@ -26,7 +26,7 @@ TARGET_LIST_FILE="$ZAP_TMP_DIR/targets"
|
|
|
|
|
[ -f "$TARGET_LIST_FILE" ] && rm -rf "$ZAP_TMP_DIR"
|
|
|
|
|
[ -f "$TARGET_LIST_FILE" ] && exit 3
|
|
|
|
|
|
|
|
|
|
CURL_TIMEOUT=5
|
|
|
|
|
CURL_TIMEOUT=7
|
|
|
|
|
CURL_MAXBODY=65536
|
|
|
|
|
CURL_NOCACHE='cache-control: no-cache'
|
|
|
|
|
CURL_NOCACHE2='pragma: no-cache'
|
|
|
|
|
@ -66,38 +66,37 @@ if [ -f /etc/openwrt_release ]; then
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
mkdir -p "$ZAP_TMP_DIR"
|
|
|
|
|
|
|
|
|
|
#echo 'Original sources: https://github.com/hyperion-cs/dpi-checkers'
|
|
|
|
|
#echo 'WEB-version: https://hyperion-cs.github.io/dpi-checkers/ru/tcp-16-20/'
|
|
|
|
|
|
|
|
|
|
TEST_SUITE='
|
|
|
|
|
{ id: "US.CF-01", provider: "🇺🇸 Cloudflare", times: 1, url: "https://img.wzstats.gg/cleaver/gunFullDisplay" },
|
|
|
|
|
{ id: "US.CF-02", provider: "🇺🇸 Cloudflare", times: 1, url: "https://genshin.jmp.blue/characters/all#" },
|
|
|
|
|
{ id: "US.CF-03", provider: "🇺🇸 Cloudflare", times: 1, url: "https://api.frankfurter.dev/v1/2000-01-01..2002-12-31" },
|
|
|
|
|
{ id: "US.CF-04", provider: "🇨🇦 Cloudflare", times: 1, url: "https://www.bigcartel.com/" },
|
|
|
|
|
{ id: "US.DO-01", provider: "🇺🇸 DigitalOcean", times: 2, url: "https://genderize.io/" },
|
|
|
|
|
{ id: "DE.HE-01", provider: "🇩🇪 Hetzner", times: 1, url: "https://j.dejure.org/jcg/doctrine/doctrine_banner.webp" },
|
|
|
|
|
{ id: "DE.HE-02", provider: "🇩🇪 Hetzner", times: 1, url: "https://accesorioscelular.com/tienda/css/plugins.css" },
|
|
|
|
|
{ id: "FI.HE-01", provider: "🇫🇮 Hetzner", times: 1, url: "https://251b5cd9.nip.io/1MB.bin" },
|
|
|
|
|
{ id: "FI.HE-02", provider: "🇫🇮 Hetzner", times: 1, url: "https://nioges.com/libs/fontawesome/webfonts/fa-solid-900.woff2" },
|
|
|
|
|
{ id: "FI.HE-03", provider: "🇫🇮 Hetzner", times: 1, url: "https://5fd8bdae.nip.io/1MB.bin" },
|
|
|
|
|
{ id: "FI.HE-04", provider: "🇫🇮 Hetzner", times: 1, url: "https://5fd8bca5.nip.io/1MB.bin" },
|
|
|
|
|
{ id: "FR.OVH-01", provider: "🇫🇷 OVH", times: 1, url: "https://eu.api.ovh.com/console/rapidoc-min.js" },
|
|
|
|
|
{ id: "FR.OVH-02", provider: "🇫🇷 OVH", times: 1, url: "https://ovh.sfx.ovh/10M.bin" },
|
|
|
|
|
{ id: "SE.OR-01", provider: "🇸🇪 Oracle", times: 1, url: "https://oracle.sfx.ovh/10M.bin" },
|
|
|
|
|
{ id: "DE.AWS-01", provider: "🇩🇪 AWS", times: 1, url: "https://www.getscope.com/assets/fonts/fa-solid-900.woff2" },
|
|
|
|
|
{ id: "US.AWS-01", provider: "🇺🇸 AWS", times: 1, url: "https://corp.kaltura.com/wp-content/cache/min/1/wp-content/themes/airfleet/dist/styles/theme.css" },
|
|
|
|
|
{ id: "US.GC-01", provider: "🇺🇸 Google Cloud", times: 1, url: "https://api.usercentrics.eu/gvl/v3/en.json" },
|
|
|
|
|
{ id: "US.FST-01", provider: "🇺🇸 Fastly", times: 1, url: "https://www.jetblue.com/footer/footer-element-es2015.js" },
|
|
|
|
|
{ id: "CA.FST-01", provider: "🇨🇦 Fastly", times: 1, url: "https://ssl.p.jwpcdn.com/player/v/8.40.5/bidding.js" },
|
|
|
|
|
{ id: "US.AKM-01", provider: "🇺🇸 Akamai", times: 1, url: "https://www.roxio.com/static/roxio/images/products/creator/nxt9/call-action-footer-bg.jpg" },
|
|
|
|
|
{ id: "PL.AKM-01", provider: "🇵🇱 Akamai", times: 1, url: "https://media-assets.stryker.com/is/image/stryker/gateway_1?$max_width_1410$" },
|
|
|
|
|
{ id: "US.CDN77-01", provider: "🇺🇸 CDN77", times: 1, url: "https://cdn.eso.org/images/banner1920/eso2520a.jpg" },
|
|
|
|
|
{ id: "FR.CNTB-01", provider: "🇫🇷 Contabo", times: 1, url: "https://xdmarineshop.gr/index.php?route=index" },
|
|
|
|
|
{ id: "NL.SW-01", provider: "🇳🇱 Scaleway", times: 1, url: "https://www.velivole.fr/img/header.jpg" },
|
|
|
|
|
{ id: "US.CNST-01", provider: "🇺🇸 Constant", times: 1, url: "https://cdn.xuansiwei.com/common/lib/font-awesome/4.7.0/fontawesome-webfont.woff2?v=4.7.0" }
|
|
|
|
|
'
|
|
|
|
|
TEST_SUITE_URL="https://hyperion-cs.github.io/dpi-checkers/ru/tcp-16-20/suite.json"
|
|
|
|
|
TEST_SUITE_FN="$ZAP_TMP_DIR/${TEST_SUITE_URL##*/}"
|
|
|
|
|
TEST_SUITE_HDR="$TEST_SUITE_FN.hdr"
|
|
|
|
|
rm -f "$TEST_SUITE_FN"
|
|
|
|
|
rm -f "$TEST_SUITE_HDR"
|
|
|
|
|
|
|
|
|
|
#echo "Download $TEST_SUITE_URL ..."
|
|
|
|
|
curl -s -L -D "$TEST_SUITE_HDR" -o "$TEST_SUITE_FN" --max-time 15 "$TEST_SUITE_URL" 2>/dev/null
|
|
|
|
|
if [ ! -f "$TEST_SUITE_HDR" ] || [ -z "$TEST_SUITE_HDR" ] ; then
|
|
|
|
|
echo "ERROR: Cannot download file \"$TEST_SUITE_URL\" (connection problem)"
|
|
|
|
|
return 17
|
|
|
|
|
fi
|
|
|
|
|
RESP_STATUS=$( cat "$TEST_SUITE_HDR" | head -n 1 | awk '{print $2}' )
|
|
|
|
|
if [ "$RESP_STATUS" != 200 ]; then
|
|
|
|
|
echo "ERROR: Cannot download file \"$TEST_SUITE_URL\" (status = $RESP_STATUS)"
|
|
|
|
|
return 18
|
|
|
|
|
fi
|
|
|
|
|
if [ ! -f "$TEST_SUITE_FN" ] || [ -z "$TEST_SUITE_FN" ] ; then
|
|
|
|
|
echo "ERROR: Cannot download file \"$TEST_SUITE_URL\" (resp body empty)"
|
|
|
|
|
return 19
|
|
|
|
|
fi
|
|
|
|
|
if [ "$(head -c 2 "$TEST_SUITE_FN" 2>/dev/null)" != "$(printf '[\n')" ]; then
|
|
|
|
|
echo "ERROR: incorrect format of \"$TEST_SUITE_URL\""
|
|
|
|
|
return 19
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ "$opt_sites" = true ]; then
|
|
|
|
|
TEST_SUITE='
|
|
|
|
|
gosuslugi.ru | @ | 40000 | https://gosuslugi.ru/__jsch/static/script.js
|
|
|
|
|
esia.gosuslugi.ru | @ | 40000 | https://esia.gosuslugi.ru/__jsch/static/script.js
|
|
|
|
|
@ -126,7 +125,12 @@ if [ "$opt_sites" = true ]; then
|
|
|
|
|
genderize.io | @# | 210000 | https://genderize.io
|
|
|
|
|
ottai.com | @ | 70000 | https://seas.static.ottai.com/ottai-website/public/images/new/home/banner/uk/banner.webp
|
|
|
|
|
'
|
|
|
|
|
|
|
|
|
|
if [ "$opt_sites" = true ]; then
|
|
|
|
|
CURL_TIMEOUT=7
|
|
|
|
|
else
|
|
|
|
|
CURL_TIMEOUT=7
|
|
|
|
|
TEST_SUITE=$( cat "$TEST_SUITE_FN" )
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
function trim
|
|
|
|
|
@ -134,8 +138,6 @@ function trim
|
|
|
|
|
echo "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
mkdir -p "$ZAP_TMP_DIR"
|
|
|
|
|
|
|
|
|
|
: > "$TARGET_LIST_FILE"
|
|
|
|
|
IDX=0
|
|
|
|
|
while IFS= read -r line; do
|
|
|
|
|
@ -151,15 +153,14 @@ while IFS= read -r line; do
|
|
|
|
|
continue
|
|
|
|
|
fi
|
|
|
|
|
case "$line" in
|
|
|
|
|
*id:*provider:*url:*)
|
|
|
|
|
*id*provider*thresholdBytes*url*)
|
|
|
|
|
IDX=$((IDX + 1))
|
|
|
|
|
TAG=$( printf '%s\n' "$line" | cut -d'"' -f2 )
|
|
|
|
|
TAG=$( printf '%s\n' "$line" | cut -d'"' -f4 )
|
|
|
|
|
COUNTRY="${TAG%%.*}"
|
|
|
|
|
PROVIDER_RAW=$( printf '%s\n' "$line" | cut -d'"' -f4 )
|
|
|
|
|
PROVIDER="${PROVIDER_RAW#* }"
|
|
|
|
|
TIMES=$( printf '%s\n' "$line" | cut -d':' -f4 | cut -d',' -f1 | tr -d ' ')
|
|
|
|
|
URL=$( printf '%s\n' "$line" | cut -d'"' -f6 )
|
|
|
|
|
echo "${IDX}|${TAG}|${COUNTRY}|${PROVIDER}|${TIMES}|${URL}" >> "$TARGET_LIST_FILE"
|
|
|
|
|
PROVIDER=$( printf '%s\n' "$line" | cut -d'"' -f8 )
|
|
|
|
|
BYTES=$( printf '%s\n' "$line" | cut -d'"' -f15 | cut -d':' -f2 | cut -d',' -f1 | tr -d ' ')
|
|
|
|
|
URL=$( printf '%s\n' "$line" | cut -d'"' -f20 )
|
|
|
|
|
echo "${IDX}|${TAG}|${COUNTRY}|${PROVIDER}|${BYTES}|${URL}" >> "$TARGET_LIST_FILE"
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
done <<EOF
|
|
|
|
|
@ -170,7 +171,7 @@ CURL_CON_TIMEOUT=$((CURL_TIMEOUT-2))
|
|
|
|
|
CURL_SPEED_TIME=$((CURL_TIMEOUT-2))
|
|
|
|
|
CURL_SPEED_LIMIT=1
|
|
|
|
|
|
|
|
|
|
while IFS='|' read -r ID TAG COUNTRY PROVIDER TIMES URL; do
|
|
|
|
|
while IFS='|' read -r ID TAG COUNTRY PROVIDER TSIZE URL; do
|
|
|
|
|
[ -z "$TAG" ] && continue
|
|
|
|
|
ID3=$( printf '%03d' "$ID" )
|
|
|
|
|
RANGETO=""
|
|
|
|
|
@ -178,14 +179,13 @@ while IFS='|' read -r ID TAG COUNTRY PROVIDER TIMES URL; do
|
|
|
|
|
USERAGENT="$CURL_USERAGENT"
|
|
|
|
|
if [ "$opt_sites" = true ]; then
|
|
|
|
|
FLAGS="$PROVIDER"
|
|
|
|
|
TSIZE="$TIMES"
|
|
|
|
|
[ "$TSIZE" = "" ] && TSIZE=$CURL_MAXBODY
|
|
|
|
|
if echo "$FLAGS" | grep -q '@'; then
|
|
|
|
|
RANGETO=""
|
|
|
|
|
else
|
|
|
|
|
RANGETO="--range 0-$((TSIZE - 1))"
|
|
|
|
|
fi
|
|
|
|
|
PROVIDER="$TSIZE"
|
|
|
|
|
PROVIDER="@"
|
|
|
|
|
if echo "$FLAGS" | grep -q '#'; then
|
|
|
|
|
REDIRECT="-L"
|
|
|
|
|
fi
|
|
|
|
|
@ -193,34 +193,47 @@ while IFS='|' read -r ID TAG COUNTRY PROVIDER TIMES URL; do
|
|
|
|
|
USERAGENT="curl/8.12"
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
RANGETO="--range 0-$((CURL_MAXBODY - 1))"
|
|
|
|
|
RANGETO="--range 0-$((TSIZE - 1))"
|
|
|
|
|
COUNTRY=$( echo "$TAG" | cut -d. -f1 )
|
|
|
|
|
CNTFLAG=$( echo "$PROVIDER" | awk '{print $1}' )
|
|
|
|
|
fi
|
|
|
|
|
URL_NO_PROTO="${URL#*://}"
|
|
|
|
|
DOMAIN="${URL_NO_PROTO%%/*}"
|
|
|
|
|
URLPATH="/${URL_NO_PROTO#*/}"
|
|
|
|
|
[ "$URLPATH" = "/$URL_NO_PROTO" ] && URLPATH="/"
|
|
|
|
|
#echo "TAG=$TAG , COUNTRY=$COUNTRY , PROVIDER=$PROVIDER , DOMAIN=$DOMAIN , URL=$URL"
|
|
|
|
|
FNAME="$ZAP_TMP_DIR/$ID3=$TAG=$PROVIDER"
|
|
|
|
|
FNAME="$ZAP_TMP_DIR/$ID3=$TAG=$PROVIDER=$TSIZE"
|
|
|
|
|
(
|
|
|
|
|
echo ">>>>> Request destination IP-addr"
|
|
|
|
|
DST_IP=
|
|
|
|
|
RESOLVE_OPT=
|
|
|
|
|
if [ "$opt_dig" != "" ]; then
|
|
|
|
|
DST_IP=$( dig +time=2 +retry=1 $OPT_DIG_DNS +short "$DOMAIN" 2>/dev/null | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | head -n1 )
|
|
|
|
|
RESP=$( dig +time=2 +retry=1 $OPT_DIG_DNS +short "$DOMAIN" 2>&1 )
|
|
|
|
|
echo "$RESP"
|
|
|
|
|
DST_IP=$( echo "$RESP" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | head -n1 )
|
|
|
|
|
else
|
|
|
|
|
CURL_TIMEOUTS="--connect-timeout 2 --max-time 3 --speed-time 3 --speed-limit 1"
|
|
|
|
|
DST_IP=$( curl -4 -I -s $CURL_TIMEOUTS -o /dev/null -w '%{remote_ip}\n' "$URL" )
|
|
|
|
|
CURL_TIMEOUTS="--connect-timeout 3 --max-time 4 --speed-time 4 --speed-limit 1"
|
|
|
|
|
RESP=$( curl -4 -I --no-progress-meter $CURL_TIMEOUTS -w '%{remote_ip}\n' "$URL" 2>&1 )
|
|
|
|
|
echo "$RESP"
|
|
|
|
|
DST_IP=$( echo "$RESP" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | tail -1 )
|
|
|
|
|
if [ -z "$DST_IP" ]; then
|
|
|
|
|
DST_IP=$( curl -4 -s $CURL_TIMEOUTS -o /dev/null -r 0-0 -w '%{remote_ip}\n' "$URL" )
|
|
|
|
|
echo "----------------------------------"
|
|
|
|
|
RESP=$( curl -4 --no-progress-meter $CURL_TIMEOUTS -r 0-0 -w '%{remote_ip}\n' "$URL" 2>&1 )
|
|
|
|
|
echo "$RESP"
|
|
|
|
|
DST_IP=$( echo "$RESP" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | tail -1 )
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
if [ "$DST_IP" = "" ]; then
|
|
|
|
|
DST_IP=$( ping -c1 "$DOMAIN" 2>/dev/null | sed -n '1s/.*(\([0-9.]*\)).*/\1/p' )
|
|
|
|
|
echo ">>>>> PING"
|
|
|
|
|
RESP=$( ping -c1 "$DOMAIN" 2>&1 )
|
|
|
|
|
echo "$RESP"
|
|
|
|
|
DST_IP=$( echo "$RESP" | sed -n '1s/.*(\([0-9.]*\)).*/\1/p' )
|
|
|
|
|
fi
|
|
|
|
|
echo ">>>>> Destination IP-addr:"
|
|
|
|
|
echo "$DST_IP"
|
|
|
|
|
[ "$DST_IP" != "" ] && RESOLVE_OPT="--resolve $DOMAIN:443:$DST_IP"
|
|
|
|
|
echo "$DST_IP" > "$FNAME.ip"
|
|
|
|
|
echo "$URL" > "$FNAME.url"
|
|
|
|
|
echo ">>>>> Download target body"
|
|
|
|
|
curl "$URL" \
|
|
|
|
|
$RESOLVE_OPT \
|
|
|
|
|
$REDIRECT \
|
|
|
|
|
@ -247,9 +260,8 @@ printf '%s\n' "$ZAP_TMP_DIR"/*.log | sort | while IFS= read -r file; do
|
|
|
|
|
ID=$( echo "$FILENAME" | cut -d= -f1)
|
|
|
|
|
TAG=$( echo "$FILENAME" | cut -d= -f2)
|
|
|
|
|
PROVIDER=$(echo "$FILENAME" | cut -d= -f3 )
|
|
|
|
|
TSIZE=$(echo "$FILENAME" | cut -d= -f4 )
|
|
|
|
|
FNAME="$ZAP_TMP_DIR/$FILENAME"
|
|
|
|
|
REQ_SIZE=$CURL_MAXBODY
|
|
|
|
|
[ "$opt_sites" = true ] && REQ_SIZE="$PROVIDER"
|
|
|
|
|
BODY_SIZE=0
|
|
|
|
|
[ -f "$FNAME.body" ] && BODY_SIZE=$( wc -c < "$FNAME.body" )
|
|
|
|
|
IPADDR="x.x.x.x"
|
|
|
|
|
@ -265,7 +277,7 @@ printf '%s\n' "$ZAP_TMP_DIR"/*.log | sort | while IFS= read -r file; do
|
|
|
|
|
elif [ ! -s "$FNAME.body" ]; then
|
|
|
|
|
status="Possibly detected"
|
|
|
|
|
else
|
|
|
|
|
if [ $BODY_SIZE -lt $REQ_SIZE ]; then
|
|
|
|
|
if [ $BODY_SIZE -lt $TSIZE ]; then
|
|
|
|
|
status="Failed (recv $BODY_SIZE bytes)"
|
|
|
|
|
res=5
|
|
|
|
|
else
|
|
|
|
|
|
