from typing import Dict, List, Any, Tuple from .result import MaigretCheckResult, SiteResult # error got as a result of completed search query class CheckError: _type = 'Unknown' _desc = '' def __init__(self, typename, desc=''): self._type = typename self._desc = desc def __str__(self): if not self._desc: return f'{self._type} error' return f'{self._type} error: {self._desc}' @property def type(self): return self._type @property def desc(self): return self._desc COMMON_ERRORS = { 'Attention Required! | Cloudflare': CheckError( 'Captcha', 'Cloudflare' ), # Prefix (no closing tag) so it matches both 'Just a moment' # and Cloudflare's real title 'Just a moment...' (with the # trailing ellipsis). The exact-match form missed the ellipsis variant, so # lightweight CF pages without the /cdn-cgi/challenge-platform script went # undetected and produced false CLAIMED on message-check sites (e.g. RealMeye). 'Just a moment': CheckError( 'Bot protection', 'Cloudflare challenge page' ), 'Please stand by, while we are checking your browser': CheckError( 'Bot protection', 'Cloudflare' ), '<span data-translate="checking_browser">Checking your browser before accessing</span>': CheckError( 'Bot protection', 'Cloudflare' ), 'This website is using a security service to protect itself from online attacks.': CheckError( 'Access denied', 'Cloudflare' ), '<title>Доступ ограничен': CheckError('Censorship', 'Rostelecom'), 'document.getElementById(\'validate_form_submit\').disabled=true': CheckError( 'Captcha', 'Mail.ru' ), 'Verifying your browser, please wait...
DDoS Protection by Blazingfast.io': CheckError( 'Bot protection', 'Blazingfast' ), '404

Мы не нашли страницу': CheckError( 'Resolving', 'MegaFon 404 page' ), 'Доступ к информационному ресурсу ограничен на основании Федерального закона': CheckError( 'Censorship', 'MGTS' ), 'Incapsula incident ID': CheckError('Bot protection', 'Incapsula'), 'Client Challenge': CheckError('Bot protection', 'Anti-bot challenge'), 'DDoS-Guard': CheckError('Bot protection', 'DDoS-Guard'), 'Сайт заблокирован хостинг-провайдером': CheckError( 'Site-specific', 'Site is disabled (Beget)' ), 'Generated by cloudfront (CloudFront)': CheckError('Request blocked', 'Cloudflare'), '/cdn-cgi/challenge-platform/h/b/orchestrate/chl_page': CheckError( 'Just a moment: bot redirect challenge', 'Cloudflare' ), } PROXY_RECOMMENDATION = ( "it's recommended to use --cloudflare-bypass or proxy, " "e.g. https://vaultproxies.net/maigret" ) ERRORS_TYPES = { 'Captcha': 'Try to switch to another IP address or to use service cookies', 'Bot protection': 'Try to switch to another IP address', 'Access denied': PROXY_RECOMMENDATION, 'Censorship': 'Switch to another internet service provider', 'Request timeout': 'Try to increase timeout or to switch to another internet service provider', 'Connecting failure': 'Check your internet connection; if only a subset of sites fails, try `-n 10` to lower parallelism', 'Connecting failure (DNS)': ( 'DNS resolution failed for most sites — Maigret\'s async DNS resolver (aiodns) could not contact a server. ' 'First, try `--dns-resolver threaded` to fall back to the system DNS resolver (often fixes this on Windows / VPN / corporate networks). ' 'If that does not help, check your internet connection, VPN, or firewall, and consider a public resolver (1.1.1.1 or 8.8.8.8)' ), 'Webgate unavailable': ( 'cloudflare_bypass is enabled but every configured solver is unreachable. ' 'Verify the URLs under `cloudflare_bypass.modules` in settings.json, and start at least one solver — ' 'most commonly FlareSolverr (`docker run -d -p 8191:8191 ghcr.io/flaresolverr/flaresolverr:latest`). ' 'Or set `cloudflare_bypass.enabled` to false in settings.json (and drop `--cloudflare-bypass`) to skip CF-protected sites' ), } ERRORS_REASONS = { 'Login required': 'Add authorization cookies through `--cookies-jar-file` (see cookies.txt)', } TEMPORARY_ERRORS_TYPES = [ 'Request timeout', 'Unknown', 'Request failed', 'Connecting failure', 'Connecting failure (DNS)', 'Webgate unavailable', 'HTTP', 'Proxy', 'Interrupted', 'Connection lost', ] THRESHOLD = 3 # percent — default threshold above which an error type is "important" # Per-error-type threshold overrides. The default 3% catches systemic issues # (Captcha, Bot protection) quickly, but for some classes a low percentage is # expected noise that does NOT mean the user has a fixable problem: # # - "Connecting failure (DNS)": a few sites in the database have stale or # dead DNS records (sites that shut down). Firing the alarm at 3% means # 3 dead domains in a 100-site batch produce a Windows/VPN troubleshooting # suggestion that is wrong for nearly every user. Wait for ≥10% before # nagging — at that rate it's clearly the user's resolver, not data rot. ERROR_THRESHOLDS: Dict[str, float] = { 'Connecting failure (DNS)': 10, } def threshold_for(err_type: str) -> float: return ERROR_THRESHOLDS.get(err_type, THRESHOLD) def is_important(err_data): return err_data['perc'] >= threshold_for(err_data['err']) def is_permanent(err_type): return err_type not in TEMPORARY_ERRORS_TYPES def detect(text): for flag, err in COMMON_ERRORS.items(): if flag in text: return err return None def solution_of(err_type) -> str: return ERRORS_TYPES.get(err_type, '') def extract_and_group(search_res: Dict[str, SiteResult]) -> List[Dict[str, Any]]: errors_counts: Dict[str, int] = {} for r in search_res.values(): if r and isinstance(r, dict) and r.get('status'): if not isinstance(r['status'], MaigretCheckResult): continue err = r['status'].error if not err: continue errors_counts[err.type] = errors_counts.get(err.type, 0) + 1 counts = [] for err, count in sorted(errors_counts.items(), key=lambda x: x[1], reverse=True): counts.append( { 'err': err, 'count': count, # Keep the RAW percentage. is_important() compares this value # against the threshold, so rounding it here can push a # sub-threshold rate up to the cutoff: e.g. 8/267 = 2.996%, # but round(2.996, 2) == 3.0, which would trip the 3% # "important" threshold for a rate that is below it. Rounding # is display-only (see notify_about_errors). 'perc': count / len(search_res) * 100, } ) return counts def notify_about_errors( search_results: Dict[str, SiteResult], query_notify, show_statistics=False ) -> List[Tuple]: """ Prepare error notifications in search results, to be displayed by the notify object. Each notification is a tuple: - ``(text, symbol)`` — plain message rendered fully bold/bright - ``(text, symbol, advice)`` — header (``text``) is bold, ``advice`` is appended in normal weight so the actionable explanation does not visually overwhelm the count line. Consumer (``notify.warning``) uses ``*tuple`` unpacking; the extra arg is optional there. Example:: [ ("Too many errors of type \"Connecting failure (DNS)\" (94.0%)", "!", "DNS resolution failed for ..."), ("Verbose error statistics:", "-"), ] """ results = [] errs = extract_and_group(search_results) was_errs_displayed = False for e in errs: if not is_important(e): continue text = f'Too many errors of type "{e["err"]}" ({round(e["perc"],2)}%)' solution = solution_of(e['err']) if solution: # Pass advice separately so notify.warning can render it in # normal weight while keeping the count line bold. results.append((text, '!', solution.capitalize())) else: results.append((text, '!')) was_errs_displayed = True if show_statistics: results.append(('Verbose error statistics:', '-')) for e in errs: text = f'{e["err"]}: {round(e["perc"],2)}%' results.append((text, '!')) if was_errs_displayed: results.append( ('You can see detailed site check errors with a flag `--print-errors`', '-') ) return results