from typing import Dict, List, Any, Tuple
from .result import MaigretCheckResult, SiteResult
# error got as a result of completed search query
class CheckError:
_type = 'Unknown'
_desc = ''
def __init__(self, typename, desc=''):
self._type = typename
self._desc = desc
def __str__(self):
if not self._desc:
return f'{self._type} error'
return f'{self._type} error: {self._desc}'
@property
def type(self):
return self._type
@property
def desc(self):
return self._desc
COMMON_ERRORS = {
'
Attention Required! | Cloudflare': CheckError(
'Captcha', 'Cloudflare'
),
# Prefix (no closing tag) so it matches both 'Just a moment'
# and Cloudflare's real title 'Just a moment...' (with the
# trailing ellipsis). The exact-match form missed the ellipsis variant, so
# lightweight CF pages without the /cdn-cgi/challenge-platform script went
# undetected and produced false CLAIMED on message-check sites (e.g. RealMeye).
'Just a moment': CheckError(
'Bot protection', 'Cloudflare challenge page'
),
'Please stand by, while we are checking your browser': CheckError(
'Bot protection', 'Cloudflare'
),
'Checking your browser before accessing': CheckError(
'Bot protection', 'Cloudflare'
),
'This website is using a security service to protect itself from online attacks.': CheckError(
'Access denied', 'Cloudflare'
),
'Доступ ограничен': CheckError('Censorship', 'Rostelecom'),
'document.getElementById(\'validate_form_submit\').disabled=true': CheckError(
'Captcha', 'Mail.ru'
),
'Verifying your browser, please wait...
DDoS Protection by Blazingfast.io': CheckError(
'Bot protection', 'Blazingfast'
),
'404Мы не нашли страницу': CheckError(
'Resolving', 'MegaFon 404 page'
),
'Доступ к информационному ресурсу ограничен на основании Федерального закона': CheckError(
'Censorship', 'MGTS'
),
'Incapsula incident ID': CheckError('Bot protection', 'Incapsula'),
'
Client Challenge': CheckError('Bot protection', 'Anti-bot challenge'),
'DDoS-Guard': CheckError('Bot protection', 'DDoS-Guard'),
'Сайт заблокирован хостинг-провайдером': CheckError(
'Site-specific', 'Site is disabled (Beget)'
),
'Generated by cloudfront (CloudFront)': CheckError('Request blocked', 'Cloudflare'),
'/cdn-cgi/challenge-platform/h/b/orchestrate/chl_page': CheckError(
'Just a moment: bot redirect challenge', 'Cloudflare'
),
}
PROXY_RECOMMENDATION = (
"it's recommended to use --cloudflare-bypass or proxy, "
"e.g. https://vaultproxies.net/maigret"
)
ERRORS_TYPES = {
'Captcha': 'Try to switch to another IP address or to use service cookies',
'Bot protection': 'Try to switch to another IP address',
'Access denied': PROXY_RECOMMENDATION,
'Censorship': 'Switch to another internet service provider',
'Request timeout': 'Try to increase timeout or to switch to another internet service provider',
'Connecting failure': 'Check your internet connection; if only a subset of sites fails, try `-n 10` to lower parallelism',
'Connecting failure (DNS)': (
'DNS resolution failed for most sites — Maigret\'s async DNS resolver (aiodns) could not contact a server. '
'First, try `--dns-resolver threaded` to fall back to the system DNS resolver (often fixes this on Windows / VPN / corporate networks). '
'If that does not help, check your internet connection, VPN, or firewall, and consider a public resolver (1.1.1.1 or 8.8.8.8)'
),
'Webgate unavailable': (
'cloudflare_bypass is enabled but every configured solver is unreachable. '
'Verify the URLs under `cloudflare_bypass.modules` in settings.json, and start at least one solver — '
'most commonly FlareSolverr (`docker run -d -p 8191:8191 ghcr.io/flaresolverr/flaresolverr:latest`). '
'Or set `cloudflare_bypass.enabled` to false in settings.json (and drop `--cloudflare-bypass`) to skip CF-protected sites'
),
}
ERRORS_REASONS = {
'Login required': 'Add authorization cookies through `--cookies-jar-file` (see cookies.txt)',
}
TEMPORARY_ERRORS_TYPES = [
'Request timeout',
'Unknown',
'Request failed',
'Connecting failure',
'Connecting failure (DNS)',
'Webgate unavailable',
'HTTP',
'Proxy',
'Interrupted',
'Connection lost',
]
THRESHOLD = 3 # percent — default threshold above which an error type is "important"
# Per-error-type threshold overrides. The default 3% catches systemic issues
# (Captcha, Bot protection) quickly, but for some classes a low percentage is
# expected noise that does NOT mean the user has a fixable problem:
#
# - "Connecting failure (DNS)": a few sites in the database have stale or
# dead DNS records (sites that shut down). Firing the alarm at 3% means
# 3 dead domains in a 100-site batch produce a Windows/VPN troubleshooting
# suggestion that is wrong for nearly every user. Wait for ≥10% before
# nagging — at that rate it's clearly the user's resolver, not data rot.
ERROR_THRESHOLDS: Dict[str, float] = {
'Connecting failure (DNS)': 10,
}
def threshold_for(err_type: str) -> float:
return ERROR_THRESHOLDS.get(err_type, THRESHOLD)
def is_important(err_data):
return err_data['perc'] >= threshold_for(err_data['err'])
def is_permanent(err_type):
return err_type not in TEMPORARY_ERRORS_TYPES
def detect(text):
for flag, err in COMMON_ERRORS.items():
if flag in text:
return err
return None
def solution_of(err_type) -> str:
return ERRORS_TYPES.get(err_type, '')
def extract_and_group(search_res: Dict[str, SiteResult]) -> List[Dict[str, Any]]:
errors_counts: Dict[str, int] = {}
for r in search_res.values():
if r and isinstance(r, dict) and r.get('status'):
if not isinstance(r['status'], MaigretCheckResult):
continue
err = r['status'].error
if not err:
continue
errors_counts[err.type] = errors_counts.get(err.type, 0) + 1
counts = []
for err, count in sorted(errors_counts.items(), key=lambda x: x[1], reverse=True):
counts.append(
{
'err': err,
'count': count,
# Keep the RAW percentage. is_important() compares this value
# against the threshold, so rounding it here can push a
# sub-threshold rate up to the cutoff: e.g. 8/267 = 2.996%,
# but round(2.996, 2) == 3.0, which would trip the 3%
# "important" threshold for a rate that is below it. Rounding
# is display-only (see notify_about_errors).
'perc': count / len(search_res) * 100,
}
)
return counts
def notify_about_errors(
search_results: Dict[str, SiteResult], query_notify, show_statistics=False
) -> List[Tuple]:
"""
Prepare error notifications in search results, to be displayed by the
notify object. Each notification is a tuple:
- ``(text, symbol)`` — plain message rendered fully bold/bright
- ``(text, symbol, advice)`` — header (``text``) is bold, ``advice`` is
appended in normal weight so the actionable explanation does not
visually overwhelm the count line. Consumer (``notify.warning``)
uses ``*tuple`` unpacking; the extra arg is optional there.
Example::
[
("Too many errors of type \"Connecting failure (DNS)\" (94.0%)",
"!", "DNS resolution failed for ..."),
("Verbose error statistics:", "-"),
]
"""
results = []
errs = extract_and_group(search_results)
was_errs_displayed = False
for e in errs:
if not is_important(e):
continue
text = f'Too many errors of type "{e["err"]}" ({round(e["perc"],2)}%)'
solution = solution_of(e['err'])
if solution:
# Pass advice separately so notify.warning can render it in
# normal weight while keeping the count line bold.
results.append((text, '!', solution.capitalize()))
else:
results.append((text, '!'))
was_errs_displayed = True
if show_statistics:
results.append(('Verbose error statistics:', '-'))
for e in errs:
text = f'{e["err"]}: {round(e["perc"],2)}%'
results.append((text, '!'))
if was_errs_displayed:
results.append(
('You can see detailed site check errors with a flag `--print-errors`', '-')
)
return results