chore: import upstream snapshot with attribution

2026-07-13 13:28:29 +08:00
commit fed8b2eed7
1531 changed files with 1107494 additions and 0 deletions
@@ -0,0 +1,185 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from application.parser.remote.crawler_loader import CrawlerLoader
+from application.parser.schema.base import Document
+
+
+class DummyResponse:
+    def __init__(self, text: str) -> None:
+        self.text = text
+
+    def raise_for_status(self) -> None:
+        return None
+
+
+def _mock_validate_url(url):
+    """Mock validate_url that allows test URLs through."""
+    from urllib.parse import urlparse
+    if not urlparse(url).scheme:
+        url = "http://" + url
+    return url
+
+
+@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
+@patch("application.parser.remote.crawler_loader.pinned_request")
+def test_load_data_crawls_same_domain_links(mock_pinned_request, mock_validate_url):
+    responses = {
+        "http://example.com": DummyResponse(
+            """
+            <html>
+                <body>
+                    <a href='/about'>About</a>
+                    <a href='https://external.com/news'>External</a>
+                </body>
+            </html>
+            """
+        ),
+        "http://example.com/about": DummyResponse("<html><body>About page</body></html>"),
+    }
+
+    def response_side_effect(_method: str, url: str, timeout=30):
+        if url not in responses:
+            raise AssertionError(f"Unexpected request for URL: {url}")
+        return responses[url]
+
+    mock_pinned_request.side_effect = response_side_effect
+
+    crawler = CrawlerLoader(limit=5)
+
+    result = crawler.load_data("http://example.com")
+
+    assert len(result) == 2
+    assert all(isinstance(doc, Document) for doc in result)
+
+    sources = {doc.extra_info.get("source") for doc in result}
+    assert sources == {"http://example.com", "http://example.com/about"}
+
+    paths = {doc.extra_info.get("file_path") for doc in result}
+    assert paths == {"index.md", "about.md"}
+
+    texts = {doc.text for doc in result}
+    assert texts == {"About\nExternal", "About page"}
+
+    assert mock_pinned_request.call_count == 2
+
+
+@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
+@patch("application.parser.remote.crawler_loader.pinned_request")
+def test_load_data_accepts_list_input_and_adds_scheme(mock_pinned_request, mock_validate_url):
+    mock_pinned_request.return_value = DummyResponse("<html><body>No links here</body></html>")
+    crawler = CrawlerLoader()
+
+    result = crawler.load_data(["example.com", "unused.com"])
+
+    mock_pinned_request.assert_called_once_with("GET", "http://example.com", timeout=30)
+
+    assert len(result) == 1
+    assert result[0].text == "No links here"
+    assert result[0].extra_info == {
+        "source": "http://example.com",
+        "file_path": "index.md",
+    }
+
+
+@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
+@patch("application.parser.remote.crawler_loader.pinned_request")
+def test_load_data_respects_limit(mock_pinned_request, mock_validate_url):
+    responses = {
+        "http://example.com": DummyResponse(
+            """
+            <html>
+                <body>
+                    <a href='/about'>About</a>
+                </body>
+            </html>
+            """
+        ),
+        "http://example.com/about": DummyResponse("<html><body>About</body></html>"),
+    }
+
+    mock_pinned_request.side_effect = lambda _method, url, timeout=30: responses[url]
+
+    crawler = CrawlerLoader(limit=1)
+
+    result = crawler.load_data("http://example.com")
+
+    assert len(result) == 1
+    assert result[0].text == "About"
+    assert mock_pinned_request.call_count == 1
+
+
+@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
+@patch("application.parser.remote.crawler_loader.logging")
+@patch("application.parser.remote.crawler_loader.pinned_request")
+def test_load_data_logs_and_skips_on_request_error(mock_pinned_request, mock_logging, mock_validate_url):
+    mock_pinned_request.side_effect = Exception("load failure")
+    crawler = CrawlerLoader()
+
+    result = crawler.load_data("http://example.com")
+
+    assert result == []
+    mock_pinned_request.assert_called_once_with("GET", "http://example.com", timeout=30)
+
+    mock_logging.error.assert_called_once()
+    message, = mock_logging.error.call_args.args
+    assert "Error processing URL http://example.com" in message
+    assert mock_logging.error.call_args.kwargs.get("exc_info") is True
+
+
+@patch("application.parser.remote.crawler_loader.validate_url")
+def test_load_data_returns_empty_on_ssrf_validation_failure(mock_validate_url):
+    """Test that SSRF validation failure returns empty list."""
+    from application.core.url_validation import SSRFError
+    mock_validate_url.side_effect = SSRFError("Access to private IP not allowed")
+
+    crawler = CrawlerLoader()
+    result = crawler.load_data("http://192.168.1.1")
+
+    assert result == []
+    mock_validate_url.assert_called_once()
+
+
+def test_url_to_virtual_path_variants():
+    crawler = CrawlerLoader()
+
+    assert crawler._url_to_virtual_path("https://docs.docsgpt.cloud/") == "index.md"
+    assert (
+        crawler._url_to_virtual_path("https://docs.docsgpt.cloud/guides/setup")
+        == "guides/setup.md"
+    )
+    assert (
+        crawler._url_to_virtual_path("https://docs.docsgpt.cloud/guides/setup/")
+        == "guides/setup.md"
+    )
+    assert crawler._url_to_virtual_path("https://example.com/page.html") == "page.md"
+
+
+# =====================================================================
+# Coverage gap tests  (lines 41-43)
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestCrawlerLoaderGaps:
+    def test_pinned_fetch_builds_document_without_webbase_loader(self):
+        """The crawler should index the response body it already fetched."""
+        from application.parser.remote.crawler_loader import CrawlerLoader
+
+        loader = CrawlerLoader(limit=5)
+        with patch(
+            "application.parser.remote.crawler_loader.validate_url",
+            return_value="https://example.com",
+        ):
+            with patch(
+                "application.parser.remote.crawler_loader.pinned_request"
+            ) as mock_pinned_request:
+                mock_response = MagicMock()
+                mock_response.status_code = 200
+                mock_response.text = "<html><body>test</body></html>"
+                mock_response.raise_for_status.return_value = None
+                mock_pinned_request.return_value = mock_response
+
+                result = loader.load_data("https://example.com")
+                assert result[0].text == "test"
@@ -0,0 +1,181 @@
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+from urllib.parse import urlparse
+
+import pytest
+import requests
+
+from application.parser.remote.crawler_markdown import CrawlerLoader
+from application.parser.schema.base import Document
+
+
+class DummyResponse:
+    def __init__(self, text):
+        self.text = text
+
+    def raise_for_status(self):
+        return None
+
+
+def _fake_extract(value: str) -> SimpleNamespace:
+    value = value.split("//")[-1]
+    host = value.split("/")[0]
+    parts = host.split(".")
+    if len(parts) >= 2:
+        domain = parts[-2]
+        suffix = parts[-1]
+    else:
+        domain = host
+        suffix = ""
+    return SimpleNamespace(domain=domain, suffix=suffix)
+
+
+def _mock_validate_url(url):
+    """Mock validate_url that allows test URLs through."""
+    if not urlparse(url).scheme:
+        url = "http://" + url
+    return url
+
+
+@pytest.fixture(autouse=True)
+def _patch_validate_url(monkeypatch):
+    monkeypatch.setattr(
+        "application.parser.remote.crawler_markdown.validate_url",
+        _mock_validate_url,
+    )
+
+
+@pytest.fixture(autouse=True)
+def _patch_tldextract(monkeypatch):
+    monkeypatch.setattr(
+        "application.parser.remote.crawler_markdown.tldextract.extract",
+        _fake_extract,
+    )
+
+
+@pytest.fixture(autouse=True)
+def _patch_markdownify(monkeypatch):
+    outputs = {}
+
+    def fake_markdownify(html, *_, **__):
+        return outputs.get(html, html)
+
+    monkeypatch.setattr(
+        "application.parser.remote.crawler_markdown.markdownify",
+        fake_markdownify,
+    )
+    return outputs
+
+
+def _patch_pinned_request(monkeypatch, side_effect):
+    """Replace pinned_request with a stub that maps URL -> response or raises."""
+    def fake_pinned_request(method, url, **_kwargs):
+        return side_effect(url)
+
+    monkeypatch.setattr(
+        "application.parser.remote.crawler_markdown.pinned_request",
+        fake_pinned_request,
+    )
+
+
+def test_load_data_filters_external_links(monkeypatch, _patch_markdownify):
+    root_html = """
+    <html><head><title>Home</title></head>
+    <body><a href="/about">About</a><a href="https://other.com">Other</a><p>Welcome</p></body>
+    </html>
+    """
+    about_html = "<html><head><title>About</title></head><body>About page</body></html>"
+
+    _patch_markdownify[root_html] = "Home Markdown"
+    _patch_markdownify[about_html] = "About Markdown"
+
+    responses = {
+        "http://example.com": DummyResponse(root_html),
+        "http://example.com/about": DummyResponse(about_html),
+    }
+
+    _patch_pinned_request(monkeypatch, lambda url: responses[url])
+
+    loader = CrawlerLoader(limit=5)
+
+    docs = loader.load_data("http://example.com")
+
+    assert len(docs) == 2
+    for doc in docs:
+        assert isinstance(doc, Document)
+        assert doc.extra_info["source"] in responses
+    texts = {doc.text for doc in docs}
+    assert texts == {"Home Markdown", "About Markdown"}
+
+
+def test_load_data_allows_subdomains(monkeypatch, _patch_markdownify):
+    root_html = """
+    <html><head><title>Home</title></head>
+    <body><a href="http://blog.example.com/post">Blog</a></body>
+    </html>
+    """
+    blog_html = "<html><head><title>Blog</title></head><body>Blog post</body></html>"
+
+    _patch_markdownify[root_html] = "Home Markdown"
+    _patch_markdownify[blog_html] = "Blog Markdown"
+
+    responses = {
+        "http://example.com": DummyResponse(root_html),
+        "http://blog.example.com/post": DummyResponse(blog_html),
+    }
+
+    _patch_pinned_request(monkeypatch, lambda url: responses[url])
+
+    loader = CrawlerLoader(limit=5, allow_subdomains=True)
+
+    docs = loader.load_data("http://example.com")
+
+    sources = {doc.extra_info["source"] for doc in docs}
+    assert "http://blog.example.com/post" in sources
+    assert len(docs) == 2
+
+
+def test_load_data_handles_fetch_errors(monkeypatch, _patch_markdownify, _patch_validate_url):
+    root_html = """
+    <html><head><title>Home</title></head>
+    <body><a href="/about">About</a></body>
+    </html>
+    """
+
+    _patch_markdownify[root_html] = "Home Markdown"
+
+    def side_effect(url):
+        if url == "http://example.com":
+            return DummyResponse(root_html)
+        raise requests.exceptions.RequestException("boom")
+
+    _patch_pinned_request(monkeypatch, side_effect)
+
+    loader = CrawlerLoader(limit=5)
+    mock_print = MagicMock()
+    monkeypatch.setattr("builtins.print", mock_print)
+
+    docs = loader.load_data("http://example.com")
+
+    assert len(docs) == 1
+    assert docs[0].text == "Home Markdown"
+    assert mock_print.called
+
+
+def test_load_data_returns_empty_on_ssrf_validation_failure(monkeypatch):
+    """Test that SSRF validation failure returns empty list."""
+    from application.core.url_validation import SSRFError
+
+    def raise_ssrf_error(url):
+        raise SSRFError("Access to private IP not allowed")
+
+    monkeypatch.setattr(
+        "application.parser.remote.crawler_markdown.validate_url",
+        raise_ssrf_error,
+    )
+
+    loader = CrawlerLoader()
+    result = loader.load_data("http://192.168.1.1")
+
+    assert result == []
+
@@ -0,0 +1,314 @@
+import base64
+import pytest
+from unittest.mock import patch, MagicMock
+import requests
+
+from application.parser.remote.github_loader import GitHubLoader
+
+
+def make_response(json_data=None, status_code=200, raise_error=None):
+    resp = MagicMock()
+    resp.status_code = status_code
+    resp.json.return_value = json_data
+    if raise_error is not None:
+        resp.raise_for_status.side_effect = raise_error
+    else:
+        resp.raise_for_status.return_value = None
+    return resp
+
+
+class TestGitHubLoaderFetchFileContent:
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_text_file_base64_decoded(self, mock_get):
+        loader = GitHubLoader()
+        content_str = "Hello from README"
+        b64 = base64.b64encode(content_str.encode("utf-8")).decode("utf-8")
+        mock_get.return_value = make_response({"encoding": "base64", "content": b64})
+
+        result = loader.fetch_file_content("owner/repo", "README.md")
+
+        assert result == content_str
+        mock_get.assert_called_once_with(
+            "https://api.github.com/repos/owner/repo/contents/README.md",
+            headers=loader.headers,
+            timeout=100,
+        )
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_binary_file_skipped(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response({"encoding": "base64", "content": "AAAA"})
+
+        result = loader.fetch_file_content("owner/repo", "image.png")
+
+        assert result is None
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_non_base64_plain_content(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response({"encoding": "", "content": "Plain text"})
+
+        result = loader.fetch_file_content("owner/repo", "file.txt")
+
+        assert result == "Plain text"
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_http_error_raises(self, mock_get):
+        loader = GitHubLoader()
+        http_err = requests.HTTPError("Not found")
+        mock_get.return_value = make_response(status_code=404, raise_error=http_err)
+
+        with pytest.raises(requests.HTTPError):
+            loader.fetch_file_content("owner/repo", "missing.txt")
+
+
+class TestGitHubLoaderFetchRepoFiles:
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_recurses_directories(self, mock_get):
+        loader = GitHubLoader()
+
+        def side_effect(url, headers=None, timeout=None):
+            if url.endswith("/contents/"):
+                return make_response([
+                    {"type": "file", "path": "README.md"},
+                    {"type": "dir", "path": "src"},
+                ])
+            elif url.endswith("/contents/src"):
+                return make_response([
+                    {"type": "file", "path": "src/main.py"},
+                    {"type": "file", "path": "src/util.py"},
+                ])
+            raise AssertionError(f"Unexpected URL: {url}")
+
+        mock_get.side_effect = side_effect
+
+        files = loader.fetch_repo_files("owner/repo", path="")
+        assert set(files) == {"README.md", "src/main.py", "src/util.py"}
+
+
+class TestGitHubLoaderLoadData:
+    def test_load_data_builds_documents_from_files(self, monkeypatch):
+        loader = GitHubLoader()
+
+        # Stub out network-dependent methods
+        monkeypatch.setattr(loader, "fetch_repo_files", lambda repo, path="": [
+            "README.md", "src/main.py"
+        ])
+
+        def fake_fetch_content(repo, file_path):
+            return f"content for {file_path}"
+
+        monkeypatch.setattr(loader, "fetch_file_content", fake_fetch_content)
+
+        docs = loader.load_data("https://github.com/owner/repo")
+
+        assert len(docs) == 2
+        assert docs[0].text == "content for README.md"
+        assert docs[0].extra_info == {
+            "title": "README.md",
+            "source": "https://github.com/owner/repo/blob/main/README.md",
+        }
+        assert docs[1].text == "content for src/main.py"
+        assert docs[1].extra_info == {
+            "title": "src/main.py",
+            "source": "https://github.com/owner/repo/blob/main/src/main.py",
+        }
+
+
+
+
+class TestGitHubLoaderIsTextFile:
+    def test_known_extension(self):
+        loader = GitHubLoader()
+        assert loader.is_text_file("app.py") is True
+        assert loader.is_text_file("data.json") is True
+
+    def test_unknown_extension_with_text_mime(self):
+        loader = GitHubLoader()
+        assert loader.is_text_file("file.xml") is True
+
+    def test_binary_file(self):
+        loader = GitHubLoader()
+        assert loader.is_text_file("image.png") is False
+
+    @patch("application.parser.remote.github_loader.mimetypes.guess_type")
+    def test_mime_fallback_text(self, mock_mime):
+        mock_mime.return_value = ("text/plain", None)
+        loader = GitHubLoader()
+        assert loader.is_text_file("unknownfile.xyz") is True
+
+
+class TestGitHubLoaderMakeRequest:
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_success(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response({"ok": True}, 200)
+        resp = loader._make_request("http://example.com")
+        assert resp.status_code == 200
+
+    @patch("application.parser.remote.github_loader.time.sleep")
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_rate_limit_retry(self, mock_get, mock_sleep):
+        loader = GitHubLoader()
+        rate_resp = MagicMock()
+        rate_resp.status_code = 403
+        rate_resp.json.return_value = {"message": "API rate limit exceeded"}
+        rate_resp.headers = {
+            "X-RateLimit-Remaining": "0",
+            "X-RateLimit-Reset": "9999999",
+        }
+        ok_resp = make_response({"ok": True}, 200)
+        mock_get.side_effect = [rate_resp, ok_resp]
+
+        resp = loader._make_request("http://example.com", max_retries=2)
+        assert resp.status_code == 200
+        mock_sleep.assert_called_once()
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_rate_limit_exhausted(self, mock_get):
+        loader = GitHubLoader()
+        rate_resp = MagicMock()
+        rate_resp.status_code = 403
+        rate_resp.json.return_value = {"message": "API rate limit exceeded"}
+        rate_resp.headers = {
+            "X-RateLimit-Remaining": "0",
+            "X-RateLimit-Reset": "9999",
+        }
+        mock_get.return_value = rate_resp
+
+        with pytest.raises(Exception, match="rate limit exceeded"):
+            loader._make_request("http://example.com", max_retries=1)
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_403_non_rate_limit(self, mock_get):
+        loader = GitHubLoader()
+        resp = MagicMock()
+        resp.status_code = 403
+        resp.json.return_value = {"message": "Forbidden - need auth"}
+        resp.headers = {"X-RateLimit-Remaining": "50", "X-RateLimit-Reset": "9999"}
+        mock_get.return_value = resp
+
+        with pytest.raises(Exception, match="GitHub API error"):
+            loader._make_request("http://example.com", max_retries=1)
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_other_error_raises(self, mock_get):
+        loader = GitHubLoader()
+        resp = make_response(
+            status_code=500,
+            raise_error=requests.HTTPError("Server Error"),
+        )
+        mock_get.return_value = resp
+
+        with pytest.raises(requests.HTTPError):
+            loader._make_request("http://example.com", max_retries=1)
+
+
+class TestGitHubLoaderFetchRepoFilesErrors:
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_api_error_message_in_dict(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response(
+            {"message": "Not Found"}, 200
+        )
+
+        with pytest.raises(Exception, match="GitHub API error"):
+            loader.fetch_repo_files("owner/repo")
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_non_list_response(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response("not a list", 200)
+
+        with pytest.raises(TypeError, match="Expected list"):
+            loader.fetch_repo_files("owner/repo")
+
+
+class TestGitHubLoaderFetchFileContentEdgeCases:
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_empty_base64_text_returns_none(self, mock_get):
+        loader = GitHubLoader()
+        b64 = base64.b64encode(b"").decode("utf-8")
+        mock_get.return_value = make_response(
+            {"encoding": "base64", "content": b64}
+        )
+        result = loader.fetch_file_content("owner/repo", "empty.py")
+        assert result is None
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_empty_non_base64_returns_none(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response(
+            {"encoding": "none", "content": "   "}
+        )
+        result = loader.fetch_file_content("owner/repo", "empty.txt")
+        assert result is None
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_decode_failure_returns_none(self, mock_get):
+        loader = GitHubLoader()
+        mock_get.return_value = make_response(
+            {"encoding": "base64", "content": "invalid!!base64"}
+        )
+        result = loader.fetch_file_content("owner/repo", "broken.py")
+        assert result is None
+
+
+class TestGitHubLoaderLoadDataSkipsNone:
+    def test_skips_binary_files(self, monkeypatch):
+        loader = GitHubLoader()
+        monkeypatch.setattr(
+            loader, "fetch_repo_files", lambda repo, path="": ["a.py", "b.png"]
+        )
+
+        def fake_content(repo, fp):
+            return "code" if fp == "a.py" else None
+
+        monkeypatch.setattr(loader, "fetch_file_content", fake_content)
+        docs = loader.load_data("https://github.com/o/r")
+        assert len(docs) == 1
+        assert docs[0].doc_id == "a.py"
+
+
+class TestGitHubLoaderRobustness:
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_fetch_repo_files_non_json_raises(self, mock_get):
+        resp = MagicMock()
+        resp.json.side_effect = ValueError("No JSON")
+        mock_get.return_value = resp
+        with pytest.raises(ValueError):
+            GitHubLoader().fetch_repo_files("owner/repo")
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_fetch_repo_files_unexpected_shape_missing_type_raises(self, mock_get):
+        # Missing 'type' in items should raise KeyError when accessed
+        mock_get.return_value = make_response([{"path": "README.md"}])
+        with pytest.raises(KeyError):
+            GitHubLoader().fetch_repo_files("owner/repo")
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_fetch_file_content_non_json_raises(self, mock_get):
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.json.side_effect = ValueError("No JSON")
+        mock_get.return_value = resp
+        with pytest.raises(ValueError):
+            GitHubLoader().fetch_file_content("owner/repo", "README.md")
+
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_fetch_file_content_unexpected_shape_missing_content_returns_none(self, mock_get):
+        # encoding indicates base64 text, but 'content' key is missing
+        # With the new code, the exception is caught and returns None (treated as binary/skipped)
+        resp = make_response({"encoding": "base64"})
+        mock_get.return_value = resp
+        result = GitHubLoader().fetch_file_content("owner/repo", "file.txt")
+        assert result is None
+
+    @patch("application.parser.remote.github_loader.base64.b64decode")
+    @patch("application.parser.remote.github_loader.requests.get")
+    def test_large_binary_skip_does_not_decode(self, mock_get, mock_b64decode):
+        # Ensure we don't attempt to decode large binary content for non-text files
+        mock_b64decode.side_effect = AssertionError("b64decode should not be called for binary files")
+        mock_get.return_value = make_response({"encoding": "base64", "content": "AAA"})
+        result = GitHubLoader().fetch_file_content("owner/repo", "bigfile.bin")
+        assert result is None
@@ -0,0 +1,83 @@
+import json
+from unittest.mock import patch, MagicMock
+import pytest
+
+from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
+
+
+class TestRedditPostsLoaderRemote:
+    def test_invalid_json_raises(self):
+        loader = RedditPostsLoaderRemote()
+        with pytest.raises(ValueError) as exc:
+            loader.load_data("not a json")
+        assert "Invalid JSON input" in str(exc.value)
+
+    def test_missing_required_fields_raises(self):
+        loader = RedditPostsLoaderRemote()
+        payload = json.dumps({"client_id": "id"})
+        with pytest.raises(ValueError) as exc:
+            loader.load_data(payload)
+        assert "Missing required fields" in str(exc.value)
+        assert "client_secret" in str(exc.value)
+
+    @patch("application.parser.remote.reddit_loader.RedditPostsLoader")
+    def test_constructs_loader_and_loads_with_defaults(self, MockRedditLoader):
+        loader = RedditPostsLoaderRemote()
+
+        instance = MagicMock()
+        docs = [MagicMock(), MagicMock()]
+        instance.load.return_value = docs
+        MockRedditLoader.return_value = instance
+
+        payload = {
+            "client_id": "cid",
+            "client_secret": "csecret",
+            "user_agent": "ua",
+            "search_queries": ["r/langchain"],
+        }
+
+        result = loader.load_data(json.dumps(payload))
+
+        MockRedditLoader.assert_called_once_with(
+            client_id="cid",
+            client_secret="csecret",
+            user_agent="ua",
+            categories=["new", "hot"],
+            mode="subreddit",
+            search_queries=["r/langchain"],
+            number_posts=10,
+        )
+        instance.load.assert_called_once()
+        assert result == docs
+
+    @patch("application.parser.remote.reddit_loader.RedditPostsLoader")
+    def test_constructs_loader_and_loads_with_overrides(self, MockRedditLoader):
+        loader = RedditPostsLoaderRemote()
+
+        instance = MagicMock()
+        instance.load.return_value = []
+        MockRedditLoader.return_value = instance
+
+        payload = {
+            "client_id": "cid",
+            "client_secret": "csecret",
+            "user_agent": "ua",
+            "search_queries": ["python"],
+            "categories": ["hot"],
+            "mode": "comments",
+            "number_posts": 3,
+        }
+
+        loader.load_data(json.dumps(payload))
+
+        MockRedditLoader.assert_called_once_with(
+            client_id="cid",
+            client_secret="csecret",
+            user_agent="ua",
+            categories=["hot"],
+            mode="comments",
+            search_queries=["python"],
+            number_posts=3,
+        )
+        instance.load.assert_called_once()
+
@@ -0,0 +1,131 @@
+"""Tests for application.parser.remote.remote_creator."""
+
+import json
+
+import pytest
+from unittest.mock import MagicMock
+
+
+@pytest.mark.unit
+class TestRemoteCreator:
+    def test_create_loader_valid_type(self):
+        """Cover line 34: returns loader instance for valid type."""
+        from application.parser.remote.remote_creator import RemoteCreator
+
+        mock_loader_cls = MagicMock()
+        original_loaders = RemoteCreator.loaders.copy()
+        RemoteCreator.loaders["url"] = mock_loader_cls
+        try:
+            RemoteCreator.create_loader("url")
+            mock_loader_cls.assert_called_once()
+        finally:
+            RemoteCreator.loaders = original_loaders
+
+    def test_create_loader_invalid_type_raises(self):
+        """Cover lines 32-33: raises ValueError for unknown type."""
+        from application.parser.remote.remote_creator import RemoteCreator
+
+        with pytest.raises(ValueError, match="No loader class found"):
+            RemoteCreator.create_loader("nonexistent_xyz")
+
+    def test_create_loader_case_insensitive(self):
+        """Cover line 31: type.lower() normalization."""
+        from application.parser.remote.remote_creator import RemoteCreator
+
+        mock_loader_cls = MagicMock()
+        original_loaders = RemoteCreator.loaders.copy()
+        RemoteCreator.loaders["sitemap"] = mock_loader_cls
+        try:
+            RemoteCreator.create_loader("SITEMAP")
+            mock_loader_cls.assert_called_once()
+        finally:
+            RemoteCreator.loaders = original_loaders
+
+
+@pytest.mark.unit
+class TestNormalizeRemoteData:
+    """``normalize_remote_data`` maps a stored JSONB ``remote_data`` value
+    back to the ``source_data`` shape each loader expects."""
+
+    def test_none_passes_through(self):
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        assert normalize_remote_data("crawler", None) is None
+
+    def test_crawler_dict_with_url_key(self):
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        result = normalize_remote_data(
+            "crawler", {"url": "https://example.com", "provider": "crawler"}
+        )
+        assert result == "https://example.com"
+
+    def test_url_dict_with_url_key(self):
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        result = normalize_remote_data("url", {"url": "https://example.com"})
+        assert result == "https://example.com"
+
+    def test_url_legacy_raw_key(self):
+        """Legacy rows wrapped a bare URL string as ``{"raw": ...}``."""
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        result = normalize_remote_data("crawler", {"raw": "https://legacy.example.com"})
+        assert result == "https://legacy.example.com"
+
+    def test_url_dict_with_urls_list(self):
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        result = normalize_remote_data(
+            "url", {"urls": ["https://a.example.com", "https://b.example.com"]}
+        )
+        assert result == ["https://a.example.com", "https://b.example.com"]
+
+    def test_github_repo_url_key(self):
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        result = normalize_remote_data(
+            "github", {"repo_url": "https://github.com/arc53/DocsGPT"}
+        )
+        assert result == "https://github.com/arc53/DocsGPT"
+
+    def test_sitemap_dict_with_url_key(self):
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        result = normalize_remote_data("sitemap", {"url": "https://example.com/sitemap.xml"})
+        assert result == "https://example.com/sitemap.xml"
+
+    def test_plain_string_url_passes_through(self):
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        assert normalize_remote_data("crawler", "https://example.com") == "https://example.com"
+
+    def test_url_dict_without_url_key_returns_none(self):
+        """A URL-type loader must never receive a dict, even a malformed one."""
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        assert normalize_remote_data("crawler", {"provider": "crawler"}) is None
+
+    def test_reddit_dict_serialized_to_json_string(self):
+        """reddit's loader runs json.loads() — it needs a JSON string."""
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        result = normalize_remote_data(
+            "reddit", {"client_id": "x", "search_queries": ["y"]}
+        )
+        assert isinstance(result, str)
+        assert json.loads(result) == {"client_id": "x", "search_queries": ["y"]}
+
+    def test_s3_dict_passes_through(self):
+        """S3Loader.load_data() accepts a dict, so it is left untouched."""
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        data = {"bucket": "b", "prefix": "k"}
+        assert normalize_remote_data("s3", data) == data
+
+    def test_json_string_remote_data_is_parsed(self):
+        """Legacy rows that stored the JSON itself as a string still resolve."""
+        from application.parser.remote.remote_creator import normalize_remote_data
+
+        result = normalize_remote_data("crawler", '{"url": "https://example.com"}')
+        assert result == "https://example.com"
@@ -0,0 +1,905 @@
+"""Tests for S3 loader implementation."""
+
+import json
+import pytest
+from unittest.mock import MagicMock, patch
+
+from botocore.exceptions import ClientError, NoCredentialsError
+
+
+@pytest.fixture
+def mock_boto3():
+    """Mock boto3 module."""
+    with patch.dict("sys.modules", {"boto3": MagicMock()}):
+        with patch("application.parser.remote.s3_loader.boto3") as mock:
+            yield mock
+
+
+@pytest.fixture
+def s3_loader(mock_boto3):
+    """Create S3Loader instance with mocked boto3."""
+    from application.parser.remote.s3_loader import S3Loader
+
+    loader = S3Loader()
+    return loader
+
+
+class TestS3LoaderInit:
+    """Test S3Loader initialization."""
+
+    def test_init_raises_import_error_when_boto3_missing(self):
+        """Should raise ImportError when boto3 is not installed."""
+        with patch("application.parser.remote.s3_loader.boto3", None):
+            from application.parser.remote.s3_loader import S3Loader
+
+            with pytest.raises(ImportError, match="boto3 is required"):
+                S3Loader()
+
+    def test_init_sets_client_to_none(self, mock_boto3):
+        """Should initialize with s3_client as None."""
+        from application.parser.remote.s3_loader import S3Loader
+
+        loader = S3Loader()
+        assert loader.s3_client is None
+
+
+class TestNormalizeEndpointUrl:
+    """Test endpoint URL normalization for S3-compatible services."""
+
+    def test_returns_unchanged_for_empty_endpoint(self, s3_loader):
+        """Should return unchanged values when endpoint_url is empty."""
+        endpoint, bucket = s3_loader._normalize_endpoint_url("", "my-bucket")
+        assert endpoint == ""
+        assert bucket == "my-bucket"
+
+    def test_returns_unchanged_for_none_endpoint(self, s3_loader):
+        """Should return unchanged values when endpoint_url is None."""
+        endpoint, bucket = s3_loader._normalize_endpoint_url(None, "my-bucket")
+        assert endpoint is None
+        assert bucket == "my-bucket"
+
+    def test_extracts_bucket_from_do_spaces_url(self, s3_loader):
+        """Should extract bucket name from DigitalOcean Spaces bucket-prefixed URL."""
+        endpoint, bucket = s3_loader._normalize_endpoint_url(
+            "https://mybucket.nyc3.digitaloceanspaces.com", ""
+        )
+        assert endpoint == "https://nyc3.digitaloceanspaces.com"
+        assert bucket == "mybucket"
+
+    def test_extracts_bucket_overrides_provided_bucket(self, s3_loader):
+        """Should use extracted bucket when it differs from provided one."""
+        endpoint, bucket = s3_loader._normalize_endpoint_url(
+            "https://mybucket.lon1.digitaloceanspaces.com", "other-bucket"
+        )
+        assert endpoint == "https://lon1.digitaloceanspaces.com"
+        assert bucket == "mybucket"
+
+    def test_keeps_provided_bucket_when_matches_extracted(self, s3_loader):
+        """Should keep bucket when provided matches extracted."""
+        endpoint, bucket = s3_loader._normalize_endpoint_url(
+            "https://mybucket.sfo3.digitaloceanspaces.com", "mybucket"
+        )
+        assert endpoint == "https://sfo3.digitaloceanspaces.com"
+        assert bucket == "mybucket"
+
+    def test_returns_unchanged_for_standard_do_endpoint(self, s3_loader):
+        """Should return unchanged for standard DO Spaces endpoint."""
+        endpoint, bucket = s3_loader._normalize_endpoint_url(
+            "https://nyc3.digitaloceanspaces.com", "my-bucket"
+        )
+        assert endpoint == "https://nyc3.digitaloceanspaces.com"
+        assert bucket == "my-bucket"
+
+    def test_returns_unchanged_for_aws_endpoint(self, s3_loader):
+        """Should return unchanged for standard AWS S3 endpoints."""
+        endpoint, bucket = s3_loader._normalize_endpoint_url(
+            "https://s3.us-east-1.amazonaws.com", "my-bucket"
+        )
+        assert endpoint == "https://s3.us-east-1.amazonaws.com"
+        assert bucket == "my-bucket"
+
+    def test_handles_minio_endpoint(self, s3_loader):
+        """Should return unchanged for MinIO endpoints."""
+        endpoint, bucket = s3_loader._normalize_endpoint_url(
+            "http://localhost:9000", "my-bucket"
+        )
+        assert endpoint == "http://localhost:9000"
+        assert bucket == "my-bucket"
+
+
+class TestInitClient:
+    """Test S3 client initialization."""
+
+    def test_init_client_creates_boto3_client(self, s3_loader, mock_boto3):
+        """Should create boto3 S3 client with provided credentials."""
+        s3_loader._init_client(
+            aws_access_key_id="test-key",
+            aws_secret_access_key="test-secret",
+            region_name="us-west-2",
+        )
+
+        mock_boto3.client.assert_called_once()
+        call_kwargs = mock_boto3.client.call_args[1]
+        assert call_kwargs["aws_access_key_id"] == "test-key"
+        assert call_kwargs["aws_secret_access_key"] == "test-secret"
+        assert call_kwargs["region_name"] == "us-west-2"
+
+    def test_init_client_with_custom_endpoint(self, s3_loader, mock_boto3):
+        """Should configure path-style addressing for custom endpoints."""
+        with patch(
+            "application.parser.remote.s3_loader.validate_url",
+            side_effect=lambda u: u,
+        ):
+            s3_loader._init_client(
+                aws_access_key_id="test-key",
+                aws_secret_access_key="test-secret",
+                region_name="us-east-1",
+                endpoint_url="https://nyc3.digitaloceanspaces.com",
+                bucket="my-bucket",
+            )
+
+        call_kwargs = mock_boto3.client.call_args[1]
+        assert call_kwargs["endpoint_url"] == "https://nyc3.digitaloceanspaces.com"
+        assert "config" in call_kwargs
+
+    def test_init_client_normalizes_do_endpoint(self, s3_loader, mock_boto3):
+        """Should normalize DigitalOcean Spaces bucket-prefixed URLs."""
+        with patch(
+            "application.parser.remote.s3_loader.validate_url",
+            side_effect=lambda u: u,
+        ):
+            corrected_bucket = s3_loader._init_client(
+                aws_access_key_id="test-key",
+                aws_secret_access_key="test-secret",
+                region_name="us-east-1",
+                endpoint_url="https://mybucket.nyc3.digitaloceanspaces.com",
+                bucket="",
+            )
+
+        assert corrected_bucket == "mybucket"
+        call_kwargs = mock_boto3.client.call_args[1]
+        assert call_kwargs["endpoint_url"] == "https://nyc3.digitaloceanspaces.com"
+
+    def test_init_client_returns_bucket_name(self, s3_loader, mock_boto3):
+        """Should return the bucket name (potentially corrected)."""
+        result = s3_loader._init_client(
+            aws_access_key_id="test-key",
+            aws_secret_access_key="test-secret",
+            region_name="us-east-1",
+            bucket="my-bucket",
+        )
+
+        assert result == "my-bucket"
+
+
+class TestIsTextFile:
+    """Test text file detection."""
+
+    def test_recognizes_common_text_extensions(self, s3_loader):
+        """Should recognize common text file extensions."""
+        text_files = [
+            "readme.txt",
+            "docs.md",
+            "config.json",
+            "data.yaml",
+            "script.py",
+            "app.js",
+            "main.go",
+            "style.css",
+            "index.html",
+        ]
+        for filename in text_files:
+            assert s3_loader.is_text_file(filename), f"{filename} should be text"
+
+    def test_rejects_binary_extensions(self, s3_loader):
+        """Should reject binary file extensions."""
+        binary_files = ["image.png", "photo.jpg", "archive.zip", "app.exe", "doc.pdf"]
+        for filename in binary_files:
+            assert not s3_loader.is_text_file(filename), f"{filename} should not be text"
+
+    def test_case_insensitive_matching(self, s3_loader):
+        """Should match extensions case-insensitively."""
+        assert s3_loader.is_text_file("README.TXT")
+        assert s3_loader.is_text_file("Config.JSON")
+        assert s3_loader.is_text_file("Script.PY")
+
+
+class TestIsSupportedDocument:
+    """Test document file detection."""
+
+    def test_recognizes_document_extensions(self, s3_loader):
+        """Should recognize document file extensions."""
+        doc_files = [
+            "report.pdf",
+            "document.docx",
+            "spreadsheet.xlsx",
+            "presentation.pptx",
+            "book.epub",
+        ]
+        for filename in doc_files:
+            assert s3_loader.is_supported_document(
+                filename
+            ), f"{filename} should be document"
+
+    def test_rejects_non_document_extensions(self, s3_loader):
+        """Should reject non-document file extensions."""
+        non_doc_files = ["image.png", "script.py", "readme.txt", "archive.zip"]
+        for filename in non_doc_files:
+            assert not s3_loader.is_supported_document(
+                filename
+            ), f"{filename} should not be document"
+
+    def test_case_insensitive_matching(self, s3_loader):
+        """Should match extensions case-insensitively."""
+        assert s3_loader.is_supported_document("Report.PDF")
+        assert s3_loader.is_supported_document("Document.DOCX")
+
+
+class TestListObjects:
+    """Test S3 object listing."""
+
+    def test_list_objects_returns_file_keys(self, s3_loader, mock_boto3):
+        """Should return list of file keys from bucket."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [
+            {
+                "Contents": [
+                    {"Key": "file1.txt"},
+                    {"Key": "file2.md"},
+                    {"Key": "folder/"},  # Directory marker, should be skipped
+                    {"Key": "folder/file3.py"},
+                ]
+            }
+        ]
+
+        result = s3_loader.list_objects("test-bucket", "")
+
+        assert result == ["file1.txt", "file2.md", "folder/file3.py"]
+        mock_client.get_paginator.assert_called_once_with("list_objects_v2")
+        paginator.paginate.assert_called_once_with(Bucket="test-bucket", Prefix="")
+
+    def test_list_objects_with_prefix(self, s3_loader):
+        """Should filter objects by prefix."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [
+            {"Contents": [{"Key": "docs/readme.md"}, {"Key": "docs/guide.txt"}]}
+        ]
+
+        result = s3_loader.list_objects("test-bucket", "docs/")
+
+        paginator.paginate.assert_called_once_with(Bucket="test-bucket", Prefix="docs/")
+        assert len(result) == 2
+
+    def test_list_objects_handles_empty_bucket(self, s3_loader):
+        """Should return empty list for empty bucket."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [{}]  # No Contents key
+
+        result = s3_loader.list_objects("test-bucket", "")
+
+        assert result == []
+
+    def test_list_objects_raises_on_no_such_bucket(self, s3_loader):
+        """Should raise exception when bucket doesn't exist."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value.__iter__ = MagicMock(
+            side_effect=ClientError(
+                {"Error": {"Code": "NoSuchBucket", "Message": "Bucket not found"}},
+                "ListObjectsV2",
+            )
+        )
+
+        with pytest.raises(Exception, match="does not exist"):
+            s3_loader.list_objects("nonexistent-bucket", "")
+
+    def test_list_objects_raises_on_access_denied(self, s3_loader):
+        """Should raise exception on access denied."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value.__iter__ = MagicMock(
+            side_effect=ClientError(
+                {"Error": {"Code": "AccessDenied", "Message": "Access denied"}},
+                "ListObjectsV2",
+            )
+        )
+
+        with pytest.raises(Exception, match="Access denied"):
+            s3_loader.list_objects("test-bucket", "")
+
+    def test_list_objects_raises_on_no_credentials(self, s3_loader):
+        """Should raise exception when credentials are missing."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value.__iter__ = MagicMock(
+            side_effect=NoCredentialsError()
+        )
+
+        with pytest.raises(Exception, match="credentials not found"):
+            s3_loader.list_objects("test-bucket", "")
+
+
+class TestGetObjectContent:
+    """Test S3 object content retrieval."""
+
+    def test_get_text_file_content(self, s3_loader):
+        """Should return decoded text content for text files."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        mock_body = MagicMock()
+        mock_body.read.return_value = b"Hello, World!"
+        mock_client.get_object.return_value = {"Body": mock_body}
+
+        result = s3_loader.get_object_content("test-bucket", "readme.txt")
+
+        assert result == "Hello, World!"
+        mock_client.get_object.assert_called_once_with(
+            Bucket="test-bucket", Key="readme.txt"
+        )
+
+    def test_skip_unsupported_file_types(self, s3_loader):
+        """Should return None for unsupported file types."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        result = s3_loader.get_object_content("test-bucket", "image.png")
+
+        assert result is None
+        mock_client.get_object.assert_not_called()
+
+    def test_skip_empty_text_files(self, s3_loader):
+        """Should return None for empty text files."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        mock_body = MagicMock()
+        mock_body.read.return_value = b"   \n\t  "
+        mock_client.get_object.return_value = {"Body": mock_body}
+
+        result = s3_loader.get_object_content("test-bucket", "empty.txt")
+
+        assert result is None
+
+    def test_returns_none_on_unicode_decode_error(self, s3_loader):
+        """Should return None when text file can't be decoded."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        mock_body = MagicMock()
+        mock_body.read.return_value = b"\xff\xfe"  # Invalid UTF-8
+        mock_client.get_object.return_value = {"Body": mock_body}
+
+        result = s3_loader.get_object_content("test-bucket", "binary.txt")
+
+        assert result is None
+
+    def test_returns_none_on_no_such_key(self, s3_loader):
+        """Should return None when object doesn't exist."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+        mock_client.get_object.side_effect = ClientError(
+            {"Error": {"Code": "NoSuchKey", "Message": "Key not found"}},
+            "GetObject",
+        )
+
+        result = s3_loader.get_object_content("test-bucket", "missing.txt")
+
+        assert result is None
+
+    def test_returns_none_on_access_denied(self, s3_loader):
+        """Should return None when access is denied."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+        mock_client.get_object.side_effect = ClientError(
+            {"Error": {"Code": "AccessDenied", "Message": "Access denied"}},
+            "GetObject",
+        )
+
+        result = s3_loader.get_object_content("test-bucket", "secret.txt")
+
+        assert result is None
+
+    def test_processes_document_files(self, s3_loader):
+        """Should process document files through parser."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        mock_body = MagicMock()
+        mock_body.read.return_value = b"PDF content"
+        mock_client.get_object.return_value = {"Body": mock_body}
+
+        with patch.object(
+            s3_loader, "_process_document", return_value="Extracted text"
+        ) as mock_process:
+            result = s3_loader.get_object_content("test-bucket", "document.pdf")
+
+        assert result == "Extracted text"
+        mock_process.assert_called_once_with(b"PDF content", "document.pdf")
+
+
+class TestLoadData:
+    """Test main load_data method."""
+
+    def test_load_data_from_dict_input(self, s3_loader, mock_boto3):
+        """Should load documents from dict input."""
+        mock_client = MagicMock()
+        mock_boto3.client.return_value = mock_client
+
+        # Setup mock paginator
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [
+            {"Contents": [{"Key": "readme.md"}, {"Key": "guide.txt"}]}
+        ]
+
+        # Setup mock get_object
+        def get_object_side_effect(Bucket, Key):
+            mock_body = MagicMock()
+            mock_body.read.return_value = f"Content of {Key}".encode()
+            return {"Body": mock_body}
+
+        mock_client.get_object.side_effect = get_object_side_effect
+
+        input_data = {
+            "aws_access_key_id": "test-key",
+            "aws_secret_access_key": "test-secret",
+            "bucket": "test-bucket",
+        }
+
+        docs = s3_loader.load_data(input_data)
+
+        assert len(docs) == 2
+        assert docs[0].text == "Content of readme.md"
+        assert docs[0].extra_info["bucket"] == "test-bucket"
+        assert docs[0].extra_info["key"] == "readme.md"
+        assert docs[0].extra_info["source"] == "s3://test-bucket/readme.md"
+
+    def test_load_data_from_json_string(self, s3_loader, mock_boto3):
+        """Should load documents from JSON string input."""
+        mock_client = MagicMock()
+        mock_boto3.client.return_value = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [{"Contents": [{"Key": "file.txt"}]}]
+
+        mock_body = MagicMock()
+        mock_body.read.return_value = b"File content"
+        mock_client.get_object.return_value = {"Body": mock_body}
+
+        input_json = json.dumps(
+            {
+                "aws_access_key_id": "test-key",
+                "aws_secret_access_key": "test-secret",
+                "bucket": "test-bucket",
+            }
+        )
+
+        docs = s3_loader.load_data(input_json)
+
+        assert len(docs) == 1
+        assert docs[0].text == "File content"
+
+    def test_load_data_with_prefix(self, s3_loader, mock_boto3):
+        """Should filter objects by prefix."""
+        mock_client = MagicMock()
+        mock_boto3.client.return_value = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [{"Contents": [{"Key": "docs/readme.md"}]}]
+
+        mock_body = MagicMock()
+        mock_body.read.return_value = b"Documentation"
+        mock_client.get_object.return_value = {"Body": mock_body}
+
+        input_data = {
+            "aws_access_key_id": "test-key",
+            "aws_secret_access_key": "test-secret",
+            "bucket": "test-bucket",
+            "prefix": "docs/",
+        }
+
+        s3_loader.load_data(input_data)
+
+        paginator.paginate.assert_called_once_with(Bucket="test-bucket", Prefix="docs/")
+
+    def test_load_data_with_custom_region(self, s3_loader, mock_boto3):
+        """Should use custom region."""
+        mock_client = MagicMock()
+        mock_boto3.client.return_value = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [{}]
+
+        input_data = {
+            "aws_access_key_id": "test-key",
+            "aws_secret_access_key": "test-secret",
+            "bucket": "test-bucket",
+            "region": "eu-west-1",
+        }
+
+        s3_loader.load_data(input_data)
+
+        call_kwargs = mock_boto3.client.call_args[1]
+        assert call_kwargs["region_name"] == "eu-west-1"
+
+    def test_load_data_with_custom_endpoint(self, s3_loader, mock_boto3):
+        """Should use custom endpoint URL."""
+        mock_client = MagicMock()
+        mock_boto3.client.return_value = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [{}]
+
+        input_data = {
+            "aws_access_key_id": "test-key",
+            "aws_secret_access_key": "test-secret",
+            "bucket": "test-bucket",
+            "endpoint_url": "https://nyc3.digitaloceanspaces.com",
+        }
+
+        with patch(
+            "application.parser.remote.s3_loader.validate_url",
+            side_effect=lambda u: u,
+        ):
+            s3_loader.load_data(input_data)
+
+        call_kwargs = mock_boto3.client.call_args[1]
+        assert call_kwargs["endpoint_url"] == "https://nyc3.digitaloceanspaces.com"
+
+    def test_load_data_raises_on_invalid_json(self, s3_loader):
+        """Should raise ValueError for invalid JSON input."""
+        with pytest.raises(ValueError, match="Invalid JSON"):
+            s3_loader.load_data("not valid json")
+
+    def test_load_data_raises_on_missing_required_fields(self, s3_loader):
+        """Should raise ValueError when required fields are missing."""
+        with pytest.raises(ValueError, match="Missing required fields"):
+            s3_loader.load_data({"aws_access_key_id": "test-key"})
+
+        with pytest.raises(ValueError, match="Missing required fields"):
+            s3_loader.load_data(
+                {"aws_access_key_id": "test-key", "aws_secret_access_key": "secret"}
+            )
+
+    def test_load_data_skips_unsupported_files(self, s3_loader, mock_boto3):
+        """Should skip unsupported file types."""
+        mock_client = MagicMock()
+        mock_boto3.client.return_value = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [
+            {
+                "Contents": [
+                    {"Key": "readme.txt"},
+                    {"Key": "image.png"},  # Unsupported
+                    {"Key": "photo.jpg"},  # Unsupported
+                ]
+            }
+        ]
+
+        def get_object_side_effect(Bucket, Key):
+            mock_body = MagicMock()
+            mock_body.read.return_value = b"Text content"
+            return {"Body": mock_body}
+
+        mock_client.get_object.side_effect = get_object_side_effect
+
+        input_data = {
+            "aws_access_key_id": "test-key",
+            "aws_secret_access_key": "test-secret",
+            "bucket": "test-bucket",
+        }
+
+        docs = s3_loader.load_data(input_data)
+
+        # Only txt file should be loaded
+        assert len(docs) == 1
+        assert docs[0].extra_info["key"] == "readme.txt"
+
+    def test_load_data_uses_corrected_bucket_from_endpoint(self, s3_loader, mock_boto3):
+        """Should use bucket name extracted from DO Spaces URL."""
+        mock_client = MagicMock()
+        mock_boto3.client.return_value = mock_client
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value = [{"Contents": [{"Key": "file.txt"}]}]
+
+        mock_body = MagicMock()
+        mock_body.read.return_value = b"Content"
+        mock_client.get_object.return_value = {"Body": mock_body}
+
+        input_data = {
+            "aws_access_key_id": "test-key",
+            "aws_secret_access_key": "test-secret",
+            "bucket": "wrong-bucket",  # Will be corrected from endpoint
+            "endpoint_url": "https://mybucket.nyc3.digitaloceanspaces.com",
+        }
+
+        with patch(
+            "application.parser.remote.s3_loader.validate_url",
+            side_effect=lambda u: u,
+        ):
+            docs = s3_loader.load_data(input_data)
+
+        # Verify bucket name was corrected
+        paginator.paginate.assert_called_once_with(Bucket="mybucket", Prefix="")
+        assert docs[0].extra_info["bucket"] == "mybucket"
+
+
+class TestProcessDocument:
+    """Test document processing."""
+
+    def test_process_document_extracts_text(self, s3_loader):
+        """Should extract text from document files."""
+        mock_doc = MagicMock()
+        mock_doc.text = "Extracted document text"
+
+        with patch(
+            "application.parser.file.bulk.SimpleDirectoryReader"
+        ) as mock_reader_class:
+            mock_reader = MagicMock()
+            mock_reader.load_data.return_value = [mock_doc]
+            mock_reader_class.return_value = mock_reader
+
+            with patch("tempfile.NamedTemporaryFile") as mock_temp:
+                mock_file = MagicMock()
+                mock_file.__enter__ = MagicMock(return_value=mock_file)
+                mock_file.__exit__ = MagicMock(return_value=False)
+                mock_file.name = "/tmp/test.pdf"
+                mock_temp.return_value = mock_file
+
+                with patch("os.path.exists", return_value=True):
+                    with patch("os.unlink"):
+                        result = s3_loader._process_document(
+                            b"PDF content", "document.pdf"
+                        )
+
+        assert result == "Extracted document text"
+
+    def test_process_document_returns_none_on_error(self, s3_loader):
+        """Should return None when document processing fails."""
+        with patch(
+            "application.parser.file.bulk.SimpleDirectoryReader"
+        ) as mock_reader_class:
+            mock_reader_class.side_effect = Exception("Parse error")
+
+            with patch("tempfile.NamedTemporaryFile") as mock_temp:
+                mock_file = MagicMock()
+                mock_file.__enter__ = MagicMock(return_value=mock_file)
+                mock_file.__exit__ = MagicMock(return_value=False)
+                mock_file.name = "/tmp/test.pdf"
+                mock_temp.return_value = mock_file
+
+                with patch("os.path.exists", return_value=True):
+                    with patch("os.unlink"):
+                        result = s3_loader._process_document(
+                            b"PDF content", "document.pdf"
+                        )
+
+        assert result is None
+
+    def test_process_document_cleans_up_temp_file(self, s3_loader):
+        """Should clean up temporary file after processing."""
+        with patch(
+            "application.parser.file.bulk.SimpleDirectoryReader"
+        ) as mock_reader_class:
+            mock_reader = MagicMock()
+            mock_reader.load_data.return_value = []
+            mock_reader_class.return_value = mock_reader
+
+            with patch("tempfile.NamedTemporaryFile") as mock_temp:
+                mock_file = MagicMock()
+                mock_file.__enter__ = MagicMock(return_value=mock_file)
+                mock_file.__exit__ = MagicMock(return_value=False)
+                mock_file.name = "/tmp/test.pdf"
+                mock_temp.return_value = mock_file
+
+                with patch("os.path.exists", return_value=True) as mock_exists:
+                    with patch("os.unlink") as mock_unlink:
+                        s3_loader._process_document(b"PDF content", "document.pdf")
+
+                        mock_exists.assert_called_with("/tmp/test.pdf")
+                        mock_unlink.assert_called_with("/tmp/test.pdf")
+
+
+class TestListObjectsAdditional:
+    """Cover lines 225, 230-232: NoSuchKey error and generic S3 error."""
+
+    def test_list_objects_raises_on_no_such_key(self, s3_loader):
+        """Cover lines 225, 230-232: NoSuchKey error on ListObjectsV2."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+        mock_client.meta.endpoint_url = "https://nyc3.digitaloceanspaces.com"
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value.__iter__ = MagicMock(
+            side_effect=ClientError(
+                {"Error": {"Code": "NoSuchKey", "Message": "No such key"}},
+                "ListObjectsV2",
+            )
+        )
+
+        with pytest.raises(Exception, match="S3 error"):
+            s3_loader.list_objects("test-bucket", "")
+
+    def test_list_objects_raises_on_generic_error(self, s3_loader):
+        """Cover line 274: generic ClientError raises."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+        mock_client.meta.endpoint_url = "https://s3.amazonaws.com"
+
+        paginator = MagicMock()
+        mock_client.get_paginator.return_value = paginator
+        paginator.paginate.return_value.__iter__ = MagicMock(
+            side_effect=ClientError(
+                {"Error": {"Code": "InternalError", "Message": "Server error"}},
+                "ListObjectsV2",
+            )
+        )
+
+        with pytest.raises(Exception, match="S3 error"):
+            s3_loader.list_objects("test-bucket", "")
+
+
+class TestGetObjectContentAdditional:
+    """Cover lines 293, 299-302: document file and generic error paths."""
+
+    def test_get_object_content_supported_document(self, s3_loader):
+        """Cover lines 293, 308-309: supported document processed."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        mock_body = MagicMock()
+        mock_body.read.return_value = b"PDF bytes"
+        mock_client.get_object.return_value = {"Body": mock_body}
+
+        with patch.object(s3_loader, "_process_document", return_value="Extracted") as mock_proc:
+            result = s3_loader.get_object_content("bucket", "doc.pdf")
+
+        assert result == "Extracted"
+        mock_proc.assert_called_once_with(b"PDF bytes", "doc.pdf")
+
+    def test_get_object_content_generic_client_error(self, s3_loader):
+        """Cover lines 299-302: generic ClientError returns None."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+        mock_client.get_object.side_effect = ClientError(
+            {"Error": {"Code": "InternalError", "Message": "Internal error"}},
+            "GetObject",
+        )
+
+        result = s3_loader.get_object_content("bucket", "file.txt")
+        assert result is None
+
+    def test_get_object_text_empty_returns_none(self, s3_loader):
+        """Cover line 293/303-304: empty text content returns None."""
+        mock_client = MagicMock()
+        s3_loader.s3_client = mock_client
+
+        mock_body = MagicMock()
+        mock_body.read.return_value = b""
+        mock_client.get_object.return_value = {"Body": mock_body}
+
+        result = s3_loader.get_object_content("bucket", "empty.txt")
+        assert result is None
+
+
+class TestNormalizeEndpointAdditional:
+    """Cover lines 13-14, 24: import handling and digitaloceanspaces.com without region."""
+
+    def test_do_spaces_no_region(self, s3_loader):
+        """Cover line 71-76: digitaloceanspaces.com without region."""
+        endpoint, bucket = s3_loader._normalize_endpoint_url(
+            "https://digitaloceanspaces.com", "my-bucket"
+        )
+        assert endpoint == "https://digitaloceanspaces.com"
+        assert bucket == "my-bucket"
+
+
+class TestProcessDocumentAdditional:
+    """Cover lines 346-348: empty documents list returns None."""
+
+    def test_process_document_empty_documents_returns_none(self, s3_loader):
+        """Cover line 347-348: no documents extracted returns None."""
+        with patch(
+            "application.parser.file.bulk.SimpleDirectoryReader"
+        ) as mock_reader_class:
+            mock_reader = MagicMock()
+            mock_reader.load_data.return_value = []
+            mock_reader_class.return_value = mock_reader
+
+            with patch("tempfile.NamedTemporaryFile") as mock_temp:
+                mock_file = MagicMock()
+                mock_file.__enter__ = MagicMock(return_value=mock_file)
+                mock_file.__exit__ = MagicMock(return_value=False)
+                mock_file.name = "/tmp/test.docx"
+                mock_temp.return_value = mock_file
+
+                with patch("os.path.exists", return_value=True):
+                    with patch("os.unlink"):
+                        result = s3_loader._process_document(
+                            b"docx content", "document.docx"
+                        )
+
+        assert result is None
+
+
+class TestSSRFValidation:
+    """Ensure user-supplied endpoint_url values cannot target internal networks."""
+
+    def test_init_client_rejects_loopback_endpoint(self, s3_loader, mock_boto3):
+        """Should refuse to initialize boto3 when endpoint points at localhost."""
+        with pytest.raises(ValueError, match="Invalid S3 endpoint_url"):
+            s3_loader._init_client(
+                aws_access_key_id="k",
+                aws_secret_access_key="s",
+                region_name="us-east-1",
+                endpoint_url="http://127.0.0.1:9000",
+                bucket="b",
+            )
+        mock_boto3.client.assert_not_called()
+
+    def test_init_client_rejects_metadata_ip(self, s3_loader, mock_boto3):
+        """Should refuse to initialize boto3 when endpoint targets cloud metadata."""
+        with pytest.raises(ValueError, match="Invalid S3 endpoint_url"):
+            s3_loader._init_client(
+                aws_access_key_id="k",
+                aws_secret_access_key="s",
+                region_name="us-east-1",
+                endpoint_url="http://169.254.169.254/",
+                bucket="b",
+            )
+        mock_boto3.client.assert_not_called()
+
+    def test_init_client_rejects_private_ip(self, s3_loader, mock_boto3):
+        """Should refuse to initialize boto3 when endpoint targets an RFC1918 host."""
+        with pytest.raises(ValueError, match="Invalid S3 endpoint_url"):
+            s3_loader._init_client(
+                aws_access_key_id="k",
+                aws_secret_access_key="s",
+                region_name="us-east-1",
+                endpoint_url="http://10.0.0.5:9000",
+                bucket="b",
+            )
+        mock_boto3.client.assert_not_called()
+
+    def test_load_data_rejects_ssrf_endpoint(self, s3_loader, mock_boto3):
+        """load_data should surface a ValueError without hitting boto3 for blocked endpoints."""
+        input_data = {
+            "aws_access_key_id": "k",
+            "aws_secret_access_key": "s",
+            "bucket": "b",
+            "endpoint_url": "http://localhost:9000",
+        }
+        with pytest.raises(ValueError, match="Invalid S3 endpoint_url"):
+            s3_loader.load_data(input_data)
+        mock_boto3.client.assert_not_called()
@@ -0,0 +1,200 @@
+"""Tests for SharePoint loader."""
+
+from unittest.mock import patch, MagicMock
+
+from application.parser.connectors.share_point.loader import SharePointLoader
+
+
+def make_response(json_data=None, status_code=200, raise_error=None):
+    resp = MagicMock()
+    resp.status_code = status_code
+    resp.json.return_value = json_data
+    resp.content = b"test content"
+    if raise_error is not None:
+        resp.raise_for_status.side_effect = raise_error
+    else:
+        resp.raise_for_status.return_value = None
+    return resp
+
+
+class TestSharePointLoaderProcessFile:
+    """Test _process_file method."""
+
+    def test_size_retrieved_from_root_level(self):
+        """Should retrieve size from root of file_metadata, not nested file object."""
+        loader = SharePointLoader.__new__(SharePointLoader)
+
+        file_metadata = {
+            "id": "test-id",
+            "name": "test.txt",
+            "createdDateTime": "2024-01-01T00:00:00Z",
+            "lastModifiedDateTime": "2024-01-01T00:00:00Z",
+            "size": 1024,
+            "file": {
+                "mimeType": "text/plain"
+            }
+        }
+
+        doc = loader._process_file(file_metadata, load_content=False)
+
+        assert doc is not None
+        assert doc.extra_info["size"] == 1024
+        assert doc.extra_info["file_name"] == "test.txt"
+        assert doc.extra_info["mime_type"] == "text/plain"
+
+    def test_size_null_when_missing(self):
+        """Should return None when size field is missing."""
+        loader = SharePointLoader.__new__(SharePointLoader)
+
+        file_metadata = {
+            "id": "test-id",
+            "name": "test.txt",
+            "createdDateTime": "2024-01-01T00:00:00Z",
+            "lastModifiedDateTime": "2024-01-01T00:00:00Z",
+            "file": {
+                "mimeType": "text/plain"
+            }
+        }
+
+        doc = loader._process_file(file_metadata, load_content=False)
+
+        assert doc is not None
+        assert doc.extra_info["size"] is None
+
+
+class TestSharePointLoaderLoadFileById:
+    """Test _load_file_by_id method."""
+
+    @patch("application.parser.connectors.share_point.loader.requests.get")
+    @patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
+    @patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
+    @patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
+    def test_load_file_by_id_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
+        """Should include size field in $select parameter."""
+        mock_get_token.return_value = {
+            "access_token": "test-token",
+            "refresh_token": "test-refresh"
+        }
+        mock_get.return_value = make_response({
+            "id": "test-id",
+            "name": "test.txt",
+            "createdDateTime": "2024-01-01T00:00:00Z",
+            "lastModifiedDateTime": "2024-01-01T00:00:00Z",
+            "size": 2048,
+            "file": {
+                "mimeType": "text/plain"
+            }
+        })
+
+        loader = SharePointLoader("test-session")
+        doc = loader._load_file_by_id("test-id", load_content=False)
+
+        assert doc is not None
+        assert doc.extra_info["size"] == 2048
+
+        call_args = mock_get.call_args
+        params = call_args[1]["params"]
+        assert "size" in params["$select"]
+
+    @patch("application.parser.connectors.share_point.loader.requests.get")
+    @patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
+    @patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
+    @patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
+    def test_load_file_by_id_returns_document_with_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
+        """Should return document with size from API response."""
+        mock_get_token.return_value = {
+            "access_token": "test-token",
+            "refresh_token": "test-refresh"
+        }
+        mock_get.return_value = make_response({
+            "id": "test-id",
+            "name": "document.pdf",
+            "createdDateTime": "2024-01-01T00:00:00Z",
+            "lastModifiedDateTime": "2024-06-15T10:30:00Z",
+            "size": 56789,
+            "file": {
+                "mimeType": "application/pdf"
+            }
+        })
+
+        loader = SharePointLoader("test-session")
+        doc = loader._load_file_by_id("test-id", load_content=False)
+
+        assert doc is not None
+        assert doc.doc_id == "test-id"
+        assert doc.extra_info["file_name"] == "document.pdf"
+        assert doc.extra_info["mime_type"] == "application/pdf"
+        assert doc.extra_info["size"] == 56789
+        assert doc.extra_info["created_time"] == "2024-01-01T00:00:00Z"
+        assert doc.extra_info["modified_time"] == "2024-06-15T10:30:00Z"
+        assert doc.extra_info["source"] == "share_point"
+
+
+class TestSharePointLoaderListItems:
+    """Test _list_items_in_parent method."""
+
+    @patch("application.parser.connectors.share_point.loader.requests.get")
+    @patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
+    @patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
+    @patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
+    def test_list_items_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
+        """Should include size field in $select parameter when listing items."""
+        mock_get_token.return_value = {
+            "access_token": "test-token",
+            "refresh_token": "test-refresh"
+        }
+        mock_get.return_value = make_response({
+            "value": [
+                {
+                    "id": "file-1",
+                    "name": "file1.txt",
+                    "createdDateTime": "2024-01-01T00:00:00Z",
+                    "lastModifiedDateTime": "2024-01-01T00:00:00Z",
+                    "size": 12345,
+                    "file": {
+                        "mimeType": "text/plain"
+                    }
+                }
+            ]
+        })
+
+        loader = SharePointLoader("test-session")
+        docs = loader._list_items_in_parent("parent-id", limit=10, load_content=False)
+
+        assert len(docs) == 1
+        assert docs[0].extra_info["size"] == 12345
+
+        call_args = mock_get.call_args
+        params = call_args[1]["params"]
+        assert "size" in params["$select"]
+
+    @patch("application.parser.connectors.share_point.loader.requests.get")
+    @patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
+    @patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
+    @patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
+    def test_list_items_folders_include_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
+        """Should include size for folders as well."""
+        mock_get_token.return_value = {
+            "access_token": "test-token",
+            "refresh_token": "test-refresh"
+        }
+        mock_get.return_value = make_response({
+            "value": [
+                {
+                    "id": "folder-1",
+                    "name": "MyFolder",
+                    "createdDateTime": "2024-01-01T00:00:00Z",
+                    "lastModifiedDateTime": "2024-01-01T00:00:00Z",
+                    "size": 0,
+                    "folder": {}
+                }
+            ]
+        })
+
+        loader = SharePointLoader("test-session")
+        docs = loader._list_items_in_parent("parent-id", limit=10, load_content=False)
+
+        assert len(docs) == 1
+        assert docs[0].extra_info["is_folder"] is True
+        assert docs[0].extra_info["size"] == 0
+
@@ -0,0 +1,332 @@
+"""Comprehensive tests for application/parser/remote/sitemap_loader.py
+
+Covers: SitemapLoader (init, load_data, _extract_urls, _is_sitemap,
+_parse_sitemap, URL validation, error handling).
+"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+import requests
+
+from application.parser.remote.sitemap_loader import SitemapLoader
+from application.parser.schema.base import Document
+
+
+# =====================================================================
+# SitemapLoader - Init
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestSitemapLoaderInit:
+
+    def test_default_limit(self):
+        loader = SitemapLoader()
+        assert loader.limit == 20
+
+    def test_custom_limit(self):
+        loader = SitemapLoader(limit=5)
+        assert loader.limit == 5
+
+    def test_has_loader_class(self):
+        loader = SitemapLoader()
+        assert not hasattr(loader, "loader")
+
+
+# =====================================================================
+# _is_sitemap
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestIsSitemap:
+
+    def test_xml_content_type(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "application/xml"}
+        response.url = "https://example.com/sitemap.xml"
+        response.text = ""
+        assert loader._is_sitemap(response) is True
+
+    def test_xml_url_extension(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/sitemap.xml"
+        response.text = ""
+        assert loader._is_sitemap(response) is True
+
+    def test_sitemapindex_in_body(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/sitemap"
+        response.text = "<sitemapindex><sitemap></sitemap></sitemapindex>"
+        assert loader._is_sitemap(response) is True
+
+    def test_urlset_in_body(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/page"
+        response.text = "<urlset><url></url></urlset>"
+        assert loader._is_sitemap(response) is True
+
+    def test_regular_page(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/about"
+        response.text = "<html><body>About us</body></html>"
+        assert loader._is_sitemap(response) is False
+
+    def test_text_xml_content_type(self):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/xml; charset=utf-8"}
+        response.url = "https://example.com/feed"
+        response.text = ""
+        assert loader._is_sitemap(response) is True
+
+
+# =====================================================================
+# _parse_sitemap
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestParseSitemap:
+
+    def test_parse_basic_sitemap(self):
+        loader = SitemapLoader()
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://example.com/page1</loc></url>
+            <url><loc>https://example.com/page2</loc></url>
+        </urlset>"""
+
+        urls = loader._parse_sitemap(sitemap_xml)
+        assert "https://example.com/page1" in urls
+        assert "https://example.com/page2" in urls
+
+    def test_parse_nested_sitemap(self):
+        loader = SitemapLoader()
+
+        parent_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+        <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <sitemap>
+                <loc>https://example.com/sitemap-child.xml</loc>
+            </sitemap>
+        </sitemapindex>"""
+
+        with patch.object(
+            loader, "_extract_urls",
+            return_value=["https://example.com/page1"]
+        ):
+            urls = loader._parse_sitemap(parent_xml)
+            assert "https://example.com/page1" in urls
+
+    def test_parse_empty_sitemap(self):
+        loader = SitemapLoader()
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        </urlset>"""
+
+        urls = loader._parse_sitemap(sitemap_xml)
+        assert urls == []
+
+    def test_parse_skips_empty_loc_entries(self):
+        """Empty or self-closing <loc> tags must not yield None URLs."""
+        loader = SitemapLoader()
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc></loc></url>
+            <url><loc>https://example.com/p</loc></url>
+            <url><loc/></url>
+        </urlset>"""
+
+        urls = loader._parse_sitemap(sitemap_xml)
+        assert urls == ["https://example.com/p"]
+
+
+# =====================================================================
+# _extract_urls
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestExtractUrls:
+
+    @patch("application.parser.remote.sitemap_loader.pinned_request")
+    def test_extract_urls_from_sitemap(self, mock_pinned_request):
+        loader = SitemapLoader()
+
+        response = MagicMock()
+        response.headers = {"Content-Type": "application/xml"}
+        response.url = "https://example.com/sitemap.xml"
+        response.text = "<urlset><url><loc>https://example.com/p</loc></url></urlset>"
+        response.content = b"""<?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://example.com/p</loc></url>
+        </urlset>"""
+        response.raise_for_status.return_value = None
+        mock_pinned_request.return_value = response
+
+        urls = loader._extract_urls("https://example.com/sitemap.xml")
+        assert "https://example.com/p" in urls
+
+    @patch("application.parser.remote.sitemap_loader.pinned_request")
+    def test_extract_urls_not_sitemap(self, mock_pinned_request):
+        loader = SitemapLoader()
+
+        response = MagicMock()
+        response.headers = {"Content-Type": "text/html"}
+        response.url = "https://example.com/page"
+        response.text = "<html>Normal page</html>"
+        response.raise_for_status.return_value = None
+        mock_pinned_request.return_value = response
+
+        urls = loader._extract_urls("https://example.com/page")
+        assert urls == ["https://example.com/page"]
+
+    @patch("application.parser.remote.sitemap_loader.pinned_request")
+    def test_extract_urls_http_error(self, mock_pinned_request):
+        loader = SitemapLoader()
+        mock_pinned_request.side_effect = requests.exceptions.HTTPError("404")
+
+        urls = loader._extract_urls("https://example.com/missing")
+        assert urls == []
+
+    @patch("application.parser.remote.sitemap_loader.pinned_request")
+    def test_extract_urls_connection_error(self, mock_pinned_request):
+        loader = SitemapLoader()
+        mock_pinned_request.side_effect = requests.exceptions.ConnectionError()
+
+        urls = loader._extract_urls("https://example.com/bad")
+        assert urls == []
+
+    def test_extract_urls_ssrf_blocked(self):
+        from application.security.safe_url import UnsafeUserUrlError
+
+        loader = SitemapLoader()
+
+        with patch(
+            "application.parser.remote.sitemap_loader.pinned_request",
+            side_effect=UnsafeUserUrlError("blocked"),
+        ):
+            urls = loader._extract_urls("http://169.254.169.254/")
+            assert urls == []
+
+
+# =====================================================================
+# load_data
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestSitemapLoaderLoadData:
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    @patch("application.parser.remote.sitemap_loader.pinned_request")
+    def test_load_data_success(self, mock_pinned_request, mock_validate):
+        loader = SitemapLoader(limit=10)
+        mock_validate.side_effect = lambda url: url
+        response = MagicMock()
+        response.text = "Page body"
+        response.raise_for_status.return_value = None
+        mock_pinned_request.return_value = response
+
+        with patch.object(
+            loader, "_extract_urls",
+            return_value=["https://example.com/page1"]
+        ):
+            docs = loader.load_data("https://example.com/sitemap.xml")
+            assert len(docs) == 1
+            assert isinstance(docs[0], Document)
+            assert docs[0].text == "Page body"
+            assert docs[0].extra_info == {"source": "https://example.com/page1"}
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    def test_load_data_no_urls(self, mock_validate):
+        loader = SitemapLoader()
+
+        with patch.object(loader, "_extract_urls", return_value=[]):
+            docs = loader.load_data("https://example.com/empty-sitemap.xml")
+            assert docs == []
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    @patch("application.parser.remote.sitemap_loader.pinned_request")
+    def test_load_data_list_input(self, mock_pinned_request, mock_validate):
+        loader = SitemapLoader()
+        mock_validate.side_effect = lambda url: url
+        response = MagicMock()
+        response.text = "List body"
+        response.raise_for_status.return_value = None
+        mock_pinned_request.return_value = response
+
+        with patch.object(
+            loader, "_extract_urls",
+            return_value=["https://example.com/page1"]
+        ):
+            docs = loader.load_data(["https://example.com/sitemap.xml"])
+            assert len(docs) == 1
+            assert docs[0].text == "List body"
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    @patch("application.parser.remote.sitemap_loader.pinned_request")
+    def test_load_data_respects_limit(self, mock_pinned_request, mock_validate):
+        loader = SitemapLoader(limit=2)
+        mock_validate.side_effect = lambda url: url
+        response = MagicMock()
+        response.text = "Limited body"
+        response.raise_for_status.return_value = None
+        mock_pinned_request.return_value = response
+
+        urls = [f"https://example.com/page{i}" for i in range(10)]
+        with patch.object(loader, "_extract_urls", return_value=urls):
+            docs = loader.load_data("https://example.com/sitemap.xml")
+            assert len(docs) == 2
+            assert mock_pinned_request.call_count == 2
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    @patch("application.parser.remote.sitemap_loader.pinned_request")
+    def test_load_data_handles_url_error(self, mock_pinned_request, mock_validate):
+        loader = SitemapLoader()
+        mock_validate.side_effect = lambda url: url
+        mock_pinned_request.side_effect = Exception("Load failed")
+
+        with patch.object(
+            loader, "_extract_urls",
+            return_value=["https://example.com/broken"]
+        ):
+            docs = loader.load_data("https://example.com/sitemap.xml")
+            assert docs == []
+
+    def test_load_data_ssrf_blocked(self):
+        from application.core.url_validation import SSRFError
+
+        loader = SitemapLoader()
+
+        with patch(
+            "application.parser.remote.sitemap_loader.validate_url",
+            side_effect=SSRFError("blocked"),
+        ):
+            docs = loader.load_data("http://169.254.169.254/")
+            assert docs == []
+
+    @patch("application.parser.remote.sitemap_loader.validate_url")
+    @patch("application.parser.remote.sitemap_loader.pinned_request")
+    def test_load_data_no_limit(self, mock_pinned_request, mock_validate):
+        loader = SitemapLoader(limit=None)
+        mock_validate.side_effect = lambda url: url
+        response = MagicMock()
+        response.text = "No limit body"
+        response.raise_for_status.return_value = None
+        mock_pinned_request.return_value = response
+
+        urls = [f"https://example.com/page{i}" for i in range(5)]
+        with patch.object(loader, "_extract_urls", return_value=urls):
+            docs = loader.load_data("https://example.com/sitemap.xml")
+            assert len(docs) == 5
@@ -0,0 +1,316 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from urllib.parse import urlparse
+
+from application.core.url_validation import SSRFError
+from application.parser.remote.web_loader import WebLoader, headers
+from application.parser.schema.base import Document
+from langchain_core.documents import Document as LCDocument
+
+
+def _mock_validate_url(url):
+    """Mock validate_url that allows test URLs through and prepends scheme like the real impl."""
+    if not urlparse(url).scheme:
+        url = "http://" + url
+    return url
+
+
+def _fake_response(html: str) -> MagicMock:
+    """Build a MagicMock that quacks like a requests.Response for our purposes."""
+    response = MagicMock()
+    response.text = html
+    response.raise_for_status.return_value = None
+    return response
+
+
+@pytest.fixture
+def web_loader():
+    return WebLoader()
+
+
+class TestWebLoaderInitialization:
+    """Test WebLoader initialization."""
+
+    def test_init_has_no_loader_attribute(self, web_loader):
+        """Post-pinned-request migration, WebLoader no longer keeps a langchain loader class."""
+        assert not hasattr(web_loader, "loader")
+
+
+class TestWebLoaderHeaders:
+    """Test WebLoader headers configuration."""
+
+    def test_headers_defined(self):
+        assert isinstance(headers, dict)
+        assert "User-Agent" in headers
+        assert "Accept" in headers
+        assert "Accept-Language" in headers
+        assert "Referer" in headers
+        assert "DNT" in headers
+        assert "Connection" in headers
+        assert "Upgrade-Insecure-Requests" in headers
+
+    def test_headers_values(self):
+        assert headers["User-Agent"] == "Mozilla/5.0"
+        assert "text/html" in headers["Accept"]
+        assert headers["Referer"] == "https://www.google.com/"
+        assert headers["DNT"] == "1"
+        assert headers["Connection"] == "keep-alive"
+
+
+class TestWebLoaderLoadData:
+    """Test WebLoader load_data method."""
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_data_single_url_string(self, mock_pinned_request, mock_validate, web_loader):
+        mock_pinned_request.return_value = _fake_response(
+            "<html lang='en'><head><title>Test Page</title></head>"
+            "<body><p>Test web page content</p></body></html>"
+        )
+
+        result = web_loader.load_data("https://example.com")
+
+        assert len(result) == 1
+        assert isinstance(result[0], Document)
+        assert result[0].text == "Test Page\nTest web page content"
+        assert result[0].extra_info == {
+            "source": "https://example.com",
+            "title": "Test Page",
+            "language": "en",
+        }
+        mock_pinned_request.assert_called_once_with(
+            "GET", "https://example.com", headers=headers, timeout=30
+        )
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_data_multiple_urls_list(self, mock_pinned_request, mock_validate, web_loader):
+        mock_pinned_request.side_effect = [
+            _fake_response("<html><body>Content from site 1</body></html>"),
+            _fake_response("<html><body>Content from site 2</body></html>"),
+        ]
+
+        urls = ["https://site1.com", "https://site2.com"]
+        result = web_loader.load_data(urls)
+
+        assert len(result) == 2
+        assert all(isinstance(doc, Document) for doc in result)
+        assert result[0].text == "Content from site 1"
+        assert result[1].text == "Content from site 2"
+        assert result[0].extra_info == {"source": "https://site1.com"}
+        assert result[1].extra_info == {"source": "https://site2.com"}
+
+        assert mock_pinned_request.call_count == 2
+        mock_pinned_request.assert_any_call(
+            "GET", "https://site1.com", headers=headers, timeout=30
+        )
+        mock_pinned_request.assert_any_call(
+            "GET", "https://site2.com", headers=headers, timeout=30
+        )
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_data_url_without_scheme(self, mock_pinned_request, mock_validate, web_loader):
+        mock_pinned_request.return_value = _fake_response(
+            "<html><body>Schemeless</body></html>"
+        )
+
+        result = web_loader.load_data("example.com")
+
+        assert len(result) == 1
+        mock_pinned_request.assert_called_once_with(
+            "GET", "http://example.com", headers=headers, timeout=30
+        )
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_data_url_with_scheme(self, mock_pinned_request, mock_validate, web_loader):
+        mock_pinned_request.return_value = _fake_response(
+            "<html><body>Schemed</body></html>"
+        )
+
+        result = web_loader.load_data("https://example.com")
+
+        assert len(result) == 1
+        mock_pinned_request.assert_called_once_with(
+            "GET", "https://example.com", headers=headers, timeout=30
+        )
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_data_skips_pages_without_title(self, mock_pinned_request, mock_validate, web_loader):
+        """A page without <title> or <html lang=...> still loads, just with bare metadata."""
+        mock_pinned_request.return_value = _fake_response("<p>Bare body</p>")
+
+        result = web_loader.load_data("https://example.com")
+
+        assert len(result) == 1
+        assert result[0].text == "Bare body"
+        assert result[0].extra_info == {"source": "https://example.com"}
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_data_extracts_title_with_nested_markup(self, mock_pinned_request, mock_validate, web_loader):
+        """<title> with nested-looking content must still produce a non-empty title.
+
+        BS4's `html.parser` differs across Python/bs4 versions: some treat
+        `<title>` as RAWTEXT (per HTML5 spec) and return the literal string
+        ``"Hello <span>World</span>"``; others parse the inner tag and return
+        ``"HelloWorld"``. The regression guarded here is that ``soup.title.string``
+        returns ``None`` in *either* case, dropping the title entirely — using
+        ``get_text`` keeps it. Assert only that the title survives.
+        """
+        mock_pinned_request.return_value = _fake_response(
+            "<html><head><title>Hello <span>World</span></title></head>"
+            "<body><p>x</p></body></html>"
+        )
+
+        result = web_loader.load_data("https://example.com")
+
+        title = result[0].extra_info.get("title")
+        assert title, "title was dropped — soup.title.string regression?"
+        assert "Hello" in title
+
+
+class TestWebLoaderErrorHandling:
+    """Test WebLoader error handling."""
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    @patch("application.parser.remote.web_loader.logging")
+    def test_load_data_single_url_error(self, mock_logging, mock_pinned_request, mock_validate, web_loader):
+        mock_pinned_request.side_effect = Exception("Network error")
+
+        result = web_loader.load_data("https://invalid-url.com")
+
+        assert result == []
+        mock_logging.error.assert_called_once()
+        error_call = mock_logging.error.call_args
+        assert "Error processing URL https://invalid-url.com" in error_call[0][0]
+        assert error_call[1]["exc_info"] is True
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    @patch("application.parser.remote.web_loader.logging")
+    def test_load_data_partial_failure(self, mock_logging, mock_pinned_request, mock_validate, web_loader):
+        mock_pinned_request.side_effect = [
+            _fake_response("<html><body>Success content</body></html>"),
+            Exception("Network error"),
+        ]
+
+        urls = ["https://good-url.com", "https://bad-url.com"]
+        result = web_loader.load_data(urls)
+
+        assert len(result) == 1
+        assert result[0].text == "Success content"
+        assert result[0].extra_info == {"source": "https://good-url.com"}
+
+        mock_logging.error.assert_called_once()
+        error_call = mock_logging.error.call_args
+        assert "Error processing URL https://bad-url.com" in error_call[0][0]
+
+
+class TestWebLoaderSSRF:
+    """Test WebLoader SSRF protection."""
+
+    @patch("application.parser.remote.web_loader.validate_url")
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_data_skips_url_failing_ssrf_validation(self, mock_pinned_request, mock_validate, web_loader):
+        """A URL that fails SSRF validation must be skipped, never reaching the fetcher."""
+        mock_validate.side_effect = SSRFError("Access to private/internal IP addresses is not allowed.")
+
+        result = web_loader.load_data("http://169.254.169.254/latest/meta-data/")
+
+        assert result == []
+        mock_validate.assert_called_once_with("http://169.254.169.254/latest/meta-data/")
+        mock_pinned_request.assert_not_called()
+
+    @patch("application.parser.remote.web_loader.validate_url")
+    @patch("application.parser.remote.web_loader.logging")
+    def test_load_data_logs_warning_on_ssrf_failure(self, mock_logging, mock_validate, web_loader):
+        mock_validate.side_effect = SSRFError("blocked")
+
+        web_loader.load_data("http://127.0.0.1")
+
+        mock_logging.warning.assert_called_once()
+        warning_msg = mock_logging.warning.call_args[0][0]
+        assert "SSRF" in warning_msg
+        assert "127.0.0.1" in warning_msg
+
+    @patch("application.parser.remote.web_loader.validate_url")
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_data_mixed_safe_and_unsafe_urls(self, mock_pinned_request, mock_validate, web_loader):
+        def validate(url):
+            if "169.254.169.254" in url:
+                raise SSRFError("metadata blocked")
+            return url
+
+        mock_validate.side_effect = validate
+        mock_pinned_request.return_value = _fake_response(
+            "<html><body>Public page</body></html>"
+        )
+
+        urls = ["https://example.com", "http://169.254.169.254/latest/meta-data/"]
+        result = web_loader.load_data(urls)
+
+        assert len(result) == 1
+        assert result[0].text == "Public page"
+        mock_pinned_request.assert_called_once_with(
+            "GET", "https://example.com", headers=headers, timeout=30
+        )
+
+
+class TestWebLoaderEdgeCases:
+    """Test WebLoader edge cases."""
+
+    def test_load_data_empty_list(self, web_loader):
+        result = web_loader.load_data([])
+        assert result == []
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_data_empty_response_body(self, mock_pinned_request, mock_validate, web_loader):
+        mock_pinned_request.return_value = _fake_response("")
+
+        result = web_loader.load_data("https://empty-page.com")
+
+        assert len(result) == 1
+        assert result[0].text == ""
+        assert result[0].extra_info == {"source": "https://empty-page.com"}
+
+    def test_url_scheme_detection(self):
+        """Test URL scheme detection logic."""
+        assert urlparse("https://example.com").scheme == "https"
+        assert urlparse("http://example.com").scheme == "http"
+        assert urlparse("ftp://example.com").scheme == "ftp"
+
+        assert urlparse("example.com").scheme == ""
+        assert urlparse("www.example.com").scheme == ""
+
+
+class TestWebLoaderIntegration:
+    """Test WebLoader integration with base class."""
+
+    def test_inherits_from_base_remote(self, web_loader):
+        from application.parser.remote.base import BaseRemote
+        assert isinstance(web_loader, BaseRemote)
+
+    def test_implements_load_data_method(self, web_loader):
+        assert hasattr(web_loader, "load_data")
+        assert callable(web_loader.load_data)
+
+    @patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
+    @patch("application.parser.remote.web_loader.pinned_request")
+    def test_load_langchain_documents_method(self, mock_pinned_request, mock_validate, web_loader):
+        mock_pinned_request.return_value = _fake_response(
+            "<html><head><title>Test Page</title></head>"
+            "<body><p>Test web page content</p></body></html>"
+        )
+
+        result = web_loader.load_langchain_documents(inputs="https://example.com")
+
+        assert len(result) == 1
+        assert isinstance(result[0], LCDocument)
+        assert "Test web page content" in result[0].page_content
+        assert result[0].metadata["source"] == "https://example.com"
+        assert result[0].metadata["title"] == "Test Page"