soxoj--maigret/maigret/errors.py

from typing import Dict, List, Any, Tuple

from .result import MaigretCheckResult, SiteResult


# error got as a result of completed search query
class CheckError:
    _type = 'Unknown'
    _desc = ''

    def __init__(self, typename, desc=''):
        self._type = typename
        self._desc = desc

    def __str__(self):
        if not self._desc:
            return f'{self._type} error'

        return f'{self._type} error: {self._desc}'

    @property
    def type(self):
        return self._type

    @property
    def desc(self):
        return self._desc


COMMON_ERRORS = {
    '<title>Attention Required! | Cloudflare</title>': CheckError(
        'Captcha', 'Cloudflare'
    ),
    # Prefix (no closing tag) so it matches both '<title>Just a moment</title>'
    # and Cloudflare's real title '<title>Just a moment...</title>' (with the
    # trailing ellipsis). The exact-match form missed the ellipsis variant, so
    # lightweight CF pages without the /cdn-cgi/challenge-platform script went
    # undetected and produced false CLAIMED on message-check sites (e.g. RealMeye).
    '<title>Just a moment': CheckError(
        'Bot protection', 'Cloudflare challenge page'
    ),
    'Please stand by, while we are checking your browser': CheckError(
        'Bot protection', 'Cloudflare'
    ),
    '<span data-translate="checking_browser">Checking your browser before accessing</span>': CheckError(
        'Bot protection', 'Cloudflare'
    ),
    'This website is using a security service to protect itself from online attacks.': CheckError(
        'Access denied', 'Cloudflare'
    ),
    '<title>Доступ ограничен</title>': CheckError('Censorship', 'Rostelecom'),
    'document.getElementById(\'validate_form_submit\').disabled=true': CheckError(
        'Captcha', 'Mail.ru'
    ),
    'Verifying your browser, please wait...<br>DDoS Protection by</font> Blazingfast.io': CheckError(
        'Bot protection', 'Blazingfast'
    ),
    '404</h1><p class="error-card__description">Мы&nbsp;не&nbsp;нашли страницу': CheckError(
        'Resolving', 'MegaFon 404 page'
    ),
    'Доступ к информационному ресурсу ограничен на основании Федерального закона': CheckError(
        'Censorship', 'MGTS'
    ),
    'Incapsula incident ID': CheckError('Bot protection', 'Incapsula'),
    '<title>Client Challenge</title>': CheckError('Bot protection', 'Anti-bot challenge'),
    '<title>DDoS-Guard</title>': CheckError('Bot protection', 'DDoS-Guard'),
    'Сайт заблокирован хостинг-провайдером': CheckError(
        'Site-specific', 'Site is disabled (Beget)'
    ),
    'Generated by cloudfront (CloudFront)': CheckError('Request blocked', 'Cloudflare'),
    '/cdn-cgi/challenge-platform/h/b/orchestrate/chl_page': CheckError(
        'Just a moment: bot redirect challenge', 'Cloudflare'
    ),
}

PROXY_RECOMMENDATION = (
    "it's recommended to use --cloudflare-bypass or proxy, "
    "e.g. https://vaultproxies.net/maigret"
)

ERRORS_TYPES = {
    'Captcha': 'Try to switch to another IP address or to use service cookies',
    'Bot protection': 'Try to switch to another IP address',
    'Access denied': PROXY_RECOMMENDATION,
    'Censorship': 'Switch to another internet service provider',
    'Request timeout': 'Try to increase timeout or to switch to another internet service provider',
    'Connecting failure': 'Check your internet connection; if only a subset of sites fails, try `-n 10` to lower parallelism',
    'Connecting failure (DNS)': (
        'DNS resolution failed for most sites — Maigret\'s async DNS resolver (aiodns) could not contact a server. '
        'First, try `--dns-resolver threaded` to fall back to the system DNS resolver (often fixes this on Windows / VPN / corporate networks). '
        'If that does not help, check your internet connection, VPN, or firewall, and consider a public resolver (1.1.1.1 or 8.8.8.8)'
    ),
    'Webgate unavailable': (
        'cloudflare_bypass is enabled but every configured solver is unreachable. '
        'Verify the URLs under `cloudflare_bypass.modules` in settings.json, and start at least one solver — '
        'most commonly FlareSolverr (`docker run -d -p 8191:8191 ghcr.io/flaresolverr/flaresolverr:latest`). '
        'Or set `cloudflare_bypass.enabled` to false in settings.json (and drop `--cloudflare-bypass`) to skip CF-protected sites'
    ),
}

ERRORS_REASONS = {
    'Login required': 'Add authorization cookies through `--cookies-jar-file` (see cookies.txt)',
}

TEMPORARY_ERRORS_TYPES = [
    'Request timeout',
    'Unknown',
    'Request failed',
    'Connecting failure',
    'Connecting failure (DNS)',
    'Webgate unavailable',
    'HTTP',
    'Proxy',
    'Interrupted',
    'Connection lost',
]

THRESHOLD = 3  # percent — default threshold above which an error type is "important"

# Per-error-type threshold overrides. The default 3% catches systemic issues
# (Captcha, Bot protection) quickly, but for some classes a low percentage is
# expected noise that does NOT mean the user has a fixable problem:
#
# - "Connecting failure (DNS)": a few sites in the database have stale or
#   dead DNS records (sites that shut down). Firing the alarm at 3% means
#   3 dead domains in a 100-site batch produce a Windows/VPN troubleshooting
#   suggestion that is wrong for nearly every user. Wait for ≥10% before
#   nagging — at that rate it's clearly the user's resolver, not data rot.
ERROR_THRESHOLDS: Dict[str, float] = {
    'Connecting failure (DNS)': 10,
}


def threshold_for(err_type: str) -> float:
    return ERROR_THRESHOLDS.get(err_type, THRESHOLD)


def is_important(err_data):
    return err_data['perc'] >= threshold_for(err_data['err'])


def is_permanent(err_type):
    return err_type not in TEMPORARY_ERRORS_TYPES


def detect(text):
    for flag, err in COMMON_ERRORS.items():
        if flag in text:
            return err
    return None


def solution_of(err_type) -> str:
    return ERRORS_TYPES.get(err_type, '')


def extract_and_group(search_res: Dict[str, SiteResult]) -> List[Dict[str, Any]]:
    errors_counts: Dict[str, int] = {}
    for r in search_res.values():
        if r and isinstance(r, dict) and r.get('status'):
            if not isinstance(r['status'], MaigretCheckResult):
                continue

            err = r['status'].error
            if not err:
                continue
            errors_counts[err.type] = errors_counts.get(err.type, 0) + 1

    counts = []
    for err, count in sorted(errors_counts.items(), key=lambda x: x[1], reverse=True):
        counts.append(
            {
                'err': err,
                'count': count,
                # Keep the RAW percentage. is_important() compares this value
                # against the threshold, so rounding it here can push a
                # sub-threshold rate up to the cutoff: e.g. 8/267 = 2.996%,
                # but round(2.996, 2) == 3.0, which would trip the 3%
                # "important" threshold for a rate that is below it. Rounding
                # is display-only (see notify_about_errors).
                'perc': count / len(search_res) * 100,
            }
        )

    return counts


def notify_about_errors(
    search_results: Dict[str, SiteResult], query_notify, show_statistics=False
) -> List[Tuple]:
    """
    Prepare error notifications in search results, to be displayed by the
    notify object. Each notification is a tuple:

    - ``(text, symbol)`` — plain message rendered fully bold/bright
    - ``(text, symbol, advice)`` — header (``text``) is bold, ``advice`` is
      appended in normal weight so the actionable explanation does not
      visually overwhelm the count line. Consumer (``notify.warning``)
      uses ``*tuple`` unpacking; the extra arg is optional there.

    Example::

        [
            ("Too many errors of type \"Connecting failure (DNS)\" (94.0%)",
             "!", "DNS resolution failed for ..."),
            ("Verbose error statistics:", "-"),
        ]
    """
    results = []

    errs = extract_and_group(search_results)
    was_errs_displayed = False
    for e in errs:
        if not is_important(e):
            continue
        text = f'Too many errors of type "{e["err"]}" ({round(e["perc"],2)}%)'
        solution = solution_of(e['err'])
        if solution:
            # Pass advice separately so notify.warning can render it in
            # normal weight while keeping the count line bold.
            results.append((text, '!', solution.capitalize()))
        else:
            results.append((text, '!'))
        was_errs_displayed = True

    if show_statistics:
        results.append(('Verbose error statistics:', '-'))
        for e in errs:
            text = f'{e["err"]}: {round(e["perc"],2)}%'
            results.append((text, '!'))

    if was_errs_displayed:
        results.append(
            ('You can see detailed site check errors with a flag `--print-errors`', '-')
        )

    return results