Test

""" Security tests for content fetcher. Tests for XSS prevention, malicious content handling, and safe URL processing. """ from unittest.mock import MagicMock, patch from local_deep_research.research_library.downloaders.html import HTMLDownloader from local_deep_research.content_fetcher import ContentFetcher from local_deep_research.content_fetcher.url_classifier import ( URLClassifier, URLType, ) class TestHTMLDownloaderSecurity: """Security tests for HTML downloader.""" @patch( "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html" ) def test_script_tags_removed(self, mock_fetch): """Test that script tags are removed from content.""" malicious_html = """ Test

Safe content here that should be included in the extracted text output.

More safe content that should also appear in the final output.

""" mock_fetch.return_value = malicious_html downloader = HTMLDownloader() result_bytes = downloader.download("https://example.com/page") assert result_bytes is not None result = result_bytes.decode("utf-8") assert "">

More article content for the reader.

""" mock_fetch.return_value = html_with_iframe downloader = HTMLDownloader() result_bytes = downloader.download("https://example.com/page") assert result_bytes is not None result = result_bytes.decode("utf-8") assert "iframe" not in result.lower() assert "malicious.com" not in result assert "phishing" not in result @patch( "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html" ) def test_form_elements_removed(self, mock_fetch): """Test that form elements are removed (prevent phishing).""" html_with_form = """

Please enter your credentials in the form below:

This is additional content after the form element.

""" mock_fetch.return_value = html_with_form downloader = HTMLDownloader() result_bytes = downloader.download("https://example.com/page") assert result_bytes is not None result = result_bytes.decode("utf-8") assert "

Content before SVG element with additional text.

Content after SVG element with additional text.

""" mock_fetch.return_value = html_with_svg downloader = HTMLDownloader() result_bytes = downloader.download("https://example.com/page") assert result_bytes is not None result = result_bytes.decode("utf-8") assert "

Article content that should be extracted properly.

More article content for testing the output.

""" mock_fetch.return_value = html_with_style downloader = HTMLDownloader() result_bytes = downloader.download("https://example.com/page") assert result_bytes is not None result = result_bytes.decode("utf-8") assert "alert(1)", "https://example.com/page?q=

", "https://example.com/path/../../../etc/passwd", "https://example.com/page\x00nullbyte", ] for url in urls: # Should not crash url_type = URLClassifier.classify(url) assert url_type is not None def test_very_long_url(self): """Test handling of very long URLs.""" long_url = "https://example.com/" + "a" * 10000 # Should not crash or hang url_type = URLClassifier.classify(long_url) assert url_type == URLType.HTML class TestContentFetcherSecurity: """Security tests for content fetcher.""" @patch( "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader" ) def test_content_size_limit(self, mock_get_downloader): """Test that very large content is truncated.""" # 100MB of content huge_content = "A" * (100 * 1024 * 1024) mock_downloader = MagicMock() mock_downloader.download_with_result.return_value = MagicMock( content=huge_content.encode("utf-8"), is_success=True, ) mock_downloader.get_metadata.return_value = {} mock_get_downloader.return_value = mock_downloader fetcher = ContentFetcher() result = fetcher.fetch("https://example.com/huge", max_length=10000) assert result["status"] == "success" assert ( len(result["content"]) <= 11000 ) # max_length + truncation message def test_no_ssrf_via_url_classification(self): """Test that URL classification doesn't make network requests.""" # These URLs should be classified without making any network requests dangerous_urls = [ "http://169.254.169.254/latest/meta-data/", # AWS metadata "http://localhost:6379/", # Redis "http://127.0.0.1:22/", # SSH "http://[::1]/admin", # IPv6 localhost "http://0.0.0.0/", # All interfaces ] for url in dangerous_urls: # Classification should work without network access url_type = URLClassifier.classify(url) assert url_type is not None class TestHTMLExtractionSafety: """Test safe text extraction from HTML.""" @patch( "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html" ) def test_null_bytes_handled(self, mock_fetch): """Test handling of null bytes in content.""" html_with_nulls = """

Content with\x00null\x00bytes and more text here.

Additional paragraph with enough content to pass.

""" mock_fetch.return_value = html_with_nulls downloader = HTMLDownloader() # Should not crash result_bytes = downloader.download("https://example.com/page") # Result should be clean assert result_bytes is None or isinstance(result_bytes, bytes) @patch( "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html" ) def test_deeply_nested_html(self, mock_fetch): """Test handling of deeply nested HTML (potential DoS).""" # Create deeply nested HTML with content depth = 100 # Reduced from 1000 for reasonable test time nested = ( "

" * depth + "

Deeply nested content that should still be extracted properly.

" + "

" * depth ) html = f"

{nested}

" mock_fetch.return_value = html downloader = HTMLDownloader() # Should not crash or hang result_bytes = downloader.download("https://example.com/page") # BeautifulSoup should handle this gracefully assert result_bytes is None or isinstance(result_bytes, bytes) @patch( "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html" ) def test_html_entity_decoding(self, mock_fetch): """Test that HTML entities are properly decoded.""" html = """

This is an article about HTML entities: <script>alert('XSS')</script>

Special characters: & "quotes" 'apostrophes' are handled.

""" mock_fetch.return_value = html downloader = HTMLDownloader() result_bytes = downloader.download("https://example.com/page") assert result_bytes is not None result = result_bytes.decode("utf-8") # Entities should be decoded to text, not executed assert "