import pytest
from unittest.mock import patch, MagicMock
from urllib.parse import urlparse
from application.core.url_validation import SSRFError
from application.parser.remote.web_loader import WebLoader, headers
from application.parser.schema.base import Document
from langchain_core.documents import Document as LCDocument
def _mock_validate_url(url):
"""Mock validate_url that allows test URLs through and prepends scheme like the real impl."""
if not urlparse(url).scheme:
url = "http://" + url
return url
def _fake_response(html: str) -> MagicMock:
"""Build a MagicMock that quacks like a requests.Response for our purposes."""
response = MagicMock()
response.text = html
response.raise_for_status.return_value = None
return response
@pytest.fixture
def web_loader():
return WebLoader()
class TestWebLoaderInitialization:
"""Test WebLoader initialization."""
def test_init_has_no_loader_attribute(self, web_loader):
"""Post-pinned-request migration, WebLoader no longer keeps a langchain loader class."""
assert not hasattr(web_loader, "loader")
class TestWebLoaderHeaders:
"""Test WebLoader headers configuration."""
def test_headers_defined(self):
assert isinstance(headers, dict)
assert "User-Agent" in headers
assert "Accept" in headers
assert "Accept-Language" in headers
assert "Referer" in headers
assert "DNT" in headers
assert "Connection" in headers
assert "Upgrade-Insecure-Requests" in headers
def test_headers_values(self):
assert headers["User-Agent"] == "Mozilla/5.0"
assert "text/html" in headers["Accept"]
assert headers["Referer"] == "https://www.google.com/"
assert headers["DNT"] == "1"
assert headers["Connection"] == "keep-alive"
class TestWebLoaderLoadData:
"""Test WebLoader load_data method."""
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
def test_load_data_single_url_string(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"Test Page"
"Test web page content
"
)
result = web_loader.load_data("https://example.com")
assert len(result) == 1
assert isinstance(result[0], Document)
assert result[0].text == "Test Page\nTest web page content"
assert result[0].extra_info == {
"source": "https://example.com",
"title": "Test Page",
"language": "en",
}
mock_pinned_request.assert_called_once_with(
"GET", "https://example.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
def test_load_data_multiple_urls_list(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.side_effect = [
_fake_response("Content from site 1"),
_fake_response("Content from site 2"),
]
urls = ["https://site1.com", "https://site2.com"]
result = web_loader.load_data(urls)
assert len(result) == 2
assert all(isinstance(doc, Document) for doc in result)
assert result[0].text == "Content from site 1"
assert result[1].text == "Content from site 2"
assert result[0].extra_info == {"source": "https://site1.com"}
assert result[1].extra_info == {"source": "https://site2.com"}
assert mock_pinned_request.call_count == 2
mock_pinned_request.assert_any_call(
"GET", "https://site1.com", headers=headers, timeout=30
)
mock_pinned_request.assert_any_call(
"GET", "https://site2.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
def test_load_data_url_without_scheme(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"Schemeless"
)
result = web_loader.load_data("example.com")
assert len(result) == 1
mock_pinned_request.assert_called_once_with(
"GET", "http://example.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
def test_load_data_url_with_scheme(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"Schemed"
)
result = web_loader.load_data("https://example.com")
assert len(result) == 1
mock_pinned_request.assert_called_once_with(
"GET", "https://example.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
def test_load_data_skips_pages_without_title(self, mock_pinned_request, mock_validate, web_loader):
"""A page without or <html lang=...> still loads, just with bare metadata."""
mock_pinned_request.return_value = _fake_response("<p>Bare body</p>")
result = web_loader.load_data("https://example.com")
assert len(result) == 1
assert result[0].text == "Bare body"
assert result[0].extra_info == {"source": "https://example.com"}
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
def test_load_data_extracts_title_with_nested_markup(self, mock_pinned_request, mock_validate, web_loader):
"""<title> with nested-looking content must still produce a non-empty title.
BS4's `html.parser` differs across Python/bs4 versions: some treat
`<title>` as RAWTEXT (per HTML5 spec) and return the literal string
``"Hello <span>World</span>"``; others parse the inner tag and return
``"HelloWorld"``. The regression guarded here is that ``soup.title.string``
returns ``None`` in *either* case, dropping the title entirely — using
``get_text`` keeps it. Assert only that the title survives.
"""
mock_pinned_request.return_value = _fake_response(
"<html><head><title>Hello <span>World</span>"
"x