wehub-resource-sync
6ede33ccdb
Build and Push Docker Images / create_manifest (web, surfsense-web, , cpu) (push) Has been cancelled
Build and Push Docker Images / finalize_release (push) Has been cancelled
Obsidian Plugin Lint / lint (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_web, cpu, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-24.04-arm, linux/arm64, arm64, , runner, false, cpu) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_web, cpu, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-latest, linux/amd64, amd64, , runner, false, cpu) (push) Has been cancelled
Build and Push Docker Images / compute_version (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cpu, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64, , production, false, cpu) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cpu, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64, , production, false, cpu) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cu126, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64, -cuda126, production, true, cuda126) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cu126, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64, -cuda126, production, true, cuda126) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cu128, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64, -cuda, production, true, cuda) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cu128, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64, -cuda, production, true, cuda) (push) Has been cancelled
Build and Push Docker Images / verify_digests (push) Has been cancelled
Build and Push Docker Images / create_manifest (backend, surfsense-backend, , cpu) (push) Has been cancelled
Build and Push Docker Images / create_manifest (backend, surfsense-backend, -cuda, cuda) (push) Has been cancelled
Build and Push Docker Images / create_manifest (backend, surfsense-backend, -cuda126, cuda126) (push) Has been cancelled
76 lines
2.6 KiB
Python
76 lines
2.6 KiB
Python
"""Unit tests for the Phase 3e block classifier (Apache-2 layer)."""
|
|
|
|
import pytest
|
|
|
|
from app.utils.crawl import BlockType, classify_block
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
class TestClassifyBlock:
|
|
def test_ok_on_plain_content(self):
|
|
assert classify_block(200, "<html><body>hello</body></html>") is BlockType.OK
|
|
|
|
def test_none_body_is_empty(self):
|
|
assert classify_block(200, None) is BlockType.EMPTY
|
|
assert classify_block(200, " ") is BlockType.EMPTY
|
|
|
|
def test_rate_limited_takes_precedence(self):
|
|
# 429 wins even if a challenge marker is present in the body.
|
|
assert (
|
|
classify_block(429, '<div class="cf-turnstile"></div>')
|
|
is BlockType.RATE_LIMITED
|
|
)
|
|
|
|
@pytest.mark.parametrize(
|
|
"html",
|
|
[
|
|
"<title>Just a moment...</title>",
|
|
"<h1>Checking your browser before accessing</h1>",
|
|
"please enable javascript and cookies to continue",
|
|
"verify you are human",
|
|
'<div id="challenge-running"></div>',
|
|
'<div id="turnstile-wrapper"></div>',
|
|
'<div class="cf-turnstile"></div>',
|
|
"blocked by ddos-guard.net",
|
|
],
|
|
)
|
|
def test_cloudflare_markers(self, html):
|
|
assert classify_block(403, html) is BlockType.CLOUDFLARE
|
|
|
|
def test_hcaptcha(self):
|
|
assert (
|
|
classify_block(200, '<div class="h-captcha"></div>')
|
|
is BlockType.CAPTCHA_HCAPTCHA
|
|
)
|
|
|
|
def test_recaptcha(self):
|
|
assert (
|
|
classify_block(200, '<div class="g-recaptcha"></div>')
|
|
is BlockType.CAPTCHA_RECAPTCHA
|
|
)
|
|
|
|
def test_datadome(self):
|
|
assert (
|
|
classify_block(403, "var dd={'host':'geo.captcha-delivery.com'}")
|
|
is BlockType.DATADOME
|
|
)
|
|
|
|
def test_kasada(self):
|
|
assert classify_block(200, "window.KPSDK.configure()") is BlockType.KASADA
|
|
|
|
def test_bot_gate_status_without_marker_is_unknown(self):
|
|
# 202/403 with no recognized challenge marker => generic blocked-ish.
|
|
assert classify_block(202, "<html><body>x</body></html>") is BlockType.UNKNOWN
|
|
assert classify_block(403, "<html><body>x</body></html>") is BlockType.UNKNOWN
|
|
|
|
def test_bot_gate_status_empty_body_is_unknown(self):
|
|
assert classify_block(403, None) is BlockType.UNKNOWN
|
|
|
|
def test_cloudflare_marker_beats_generic_403(self):
|
|
# A 403 Cloudflare page is CLOUDFLARE, not UNKNOWN.
|
|
assert (
|
|
classify_block(403, "<title>Just a moment...</title>")
|
|
is BlockType.CLOUDFLARE
|
|
)
|