7a0da7932b
Backwards Compatibility / Verify Encryption Constants (push) Waiting to run
Backwards Compatibility / PyPI Version Compatibility (push) Waiting to run
Backwards Compatibility / Database Migration Tests (push) Waiting to run
CodeQL Advanced / Analyze (javascript-typescript) (push) Waiting to run
CodeQL Advanced / Analyze (python) (push) Waiting to run
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-form] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-metrics] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-workflow] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-core] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-pages] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [history-news] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [library] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [link-analytics] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [mobile] (push) Blocked by required conditions
Docker Tests (Consolidated) / detect-changes (push) Waiting to run
Docker Tests (Consolidated) / Build Test Image (push) Waiting to run
Docker Tests (Consolidated) / All Pytest Tests + Coverage (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [accessibility] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [api-crud] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-login] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-pages] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-register] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-core] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-lifecycle] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [error-benchmark] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) (push) Blocked by required conditions
Docker Tests (Consolidated) / Accessibility Tests (push) Blocked by required conditions
Docker Tests (Consolidated) / LLM Unit Tests (push) Blocked by required conditions
Docker Tests (Consolidated) / LLM Example Tests (push) Blocked by required conditions
Docker Tests (Consolidated) / Production Image Smoke Test (push) Blocked by required conditions
Docker Tests (Consolidated) / Infrastructure Tests (push) Blocked by required conditions
OSSF Scorecard / OSSF Security Scorecard Analysis (push) Waiting to run
OSV-Scanner (Scheduled) / scan-scheduled (push) Failing after 0s
Create Release / test-gate (push) Has been cancelled
Create Release / release-gate (push) Has been cancelled
Create Release / ci-gate (push) Has been cancelled
Create Release / version-check (push) Has been cancelled
Create Release / e2e-test-gate (push) Has been cancelled
Create Release / responsive-test-gate (push) Has been cancelled
Create Release / compat-test-gate (push) Has been cancelled
Create Release / compose-integration-gate (push) Has been cancelled
Create Release / vulture-gate (push) Has been cancelled
Create Release / build (push) Has been cancelled
Create Release / provenance (push) Has been cancelled
Create Release / prerelease-docker (push) Has been cancelled
Create Release / publish-docker (push) Has been cancelled
Create Release / create-release (push) Has been cancelled
Create Release / cleanup-changelog (push) Has been cancelled
Create Release / trigger-pypi (push) Has been cancelled
Create Release / monitor-pypi (push) Has been cancelled
Create Release / Clean up orphan prerelease tags and signatures (push) Has been cancelled
1010 lines
34 KiB
Python
1010 lines
34 KiB
Python
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import Mock
|
|
|
|
import pytest
|
|
|
|
# Handle import paths for testing
|
|
sys.path.append(str(Path(__file__).parent.parent))
|
|
from local_deep_research.report_generator import (
|
|
IntegratedReportGenerator,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_search_system():
|
|
"""Create a mock search system for testing."""
|
|
mock = Mock()
|
|
mock.analyze_topic.return_value = {
|
|
"findings": [{"content": "Test finding"}],
|
|
"current_knowledge": "Test knowledge",
|
|
"iterations": 1,
|
|
"questions_by_iteration": {1: ["Question 1?", "Question 2?"]},
|
|
}
|
|
mock.all_links_of_system = [
|
|
{"title": "Source 1", "link": "https://example.com/1"},
|
|
{"title": "Source 2", "link": "https://example.com/2"},
|
|
]
|
|
mock.strategy.settings_snapshot = {"search.iterations": 3}
|
|
mock.strategy.max_iterations = 3
|
|
return mock
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_findings():
|
|
"""Sample findings for testing."""
|
|
return {
|
|
"findings": [
|
|
{"content": "Finding 1 about AI research"},
|
|
{"content": "Finding 2 about machine learning applications"},
|
|
],
|
|
"current_knowledge": "AI research has made significant progress in recent years with applications in various fields.",
|
|
"iterations": 2,
|
|
"questions_by_iteration": {
|
|
1: [
|
|
"What are the latest advances in AI?",
|
|
"How is AI applied in healthcare?",
|
|
],
|
|
2: [
|
|
"What ethical concerns exist in AI development?",
|
|
"What is the future of AI research?",
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def report_generator(mock_llm, mock_search_system, monkeypatch):
|
|
"""Create a report generator for testing."""
|
|
monkeypatch.setattr(
|
|
"local_deep_research.report_generator.get_llm", lambda: mock_llm
|
|
)
|
|
generator = IntegratedReportGenerator(search_system=mock_search_system)
|
|
return generator
|
|
|
|
|
|
def test_init(mock_llm, mock_search_system, monkeypatch):
|
|
"""Test initialization of report generator."""
|
|
monkeypatch.setattr(
|
|
"local_deep_research.report_generator.get_llm", lambda: mock_llm
|
|
)
|
|
|
|
# Test with provided search system
|
|
generator = IntegratedReportGenerator(search_system=mock_search_system)
|
|
# Check that a model was set (might be wrapped)
|
|
assert generator.model is not None
|
|
assert generator.search_system == mock_search_system
|
|
|
|
# Test with default search system
|
|
mock_system_class = Mock()
|
|
mock_system_instance = Mock()
|
|
mock_system_class.return_value = mock_system_instance
|
|
|
|
monkeypatch.setattr(
|
|
"local_deep_research.report_generator.AdvancedSearchSystem",
|
|
mock_system_class,
|
|
)
|
|
|
|
generator = IntegratedReportGenerator()
|
|
# Check that a model was set (might be wrapped)
|
|
assert generator.model is not None
|
|
assert generator.search_system == mock_system_instance
|
|
|
|
|
|
def test_determine_report_structure(report_generator, sample_findings):
|
|
"""Test determining report structure from findings."""
|
|
# Mock the LLM response to return a specific structure
|
|
structured_response = """
|
|
STRUCTURE
|
|
1. Introduction
|
|
- Background | Provides historical context of the research topic
|
|
- Significance | Explains why this research matters
|
|
2. Key Findings
|
|
- Recent Advances | Summarizes the latest developments
|
|
- Applications | Describes how the technology is being applied
|
|
3. Discussion
|
|
- Challenges | Identifies current limitations and obstacles
|
|
- Future Directions | Explores potential future developments
|
|
END_STRUCTURE
|
|
"""
|
|
report_generator.model.invoke.return_value = Mock(
|
|
content=structured_response
|
|
)
|
|
|
|
# Call the method
|
|
structure = report_generator._determine_report_structure(
|
|
sample_findings, "AI research advances"
|
|
)
|
|
|
|
# Verify structure was parsed correctly
|
|
assert len(structure) == 3
|
|
assert structure[0]["name"] == "Introduction"
|
|
assert len(structure[0]["subsections"]) == 2
|
|
assert structure[0]["subsections"][0]["name"] == "Background"
|
|
assert (
|
|
structure[0]["subsections"][0]["purpose"]
|
|
== "Provides historical context of the research topic"
|
|
)
|
|
|
|
assert structure[1]["name"] == "Key Findings"
|
|
assert structure[2]["name"] == "Discussion"
|
|
|
|
# Verify LLM was called (can't check exact args if wrapped)
|
|
assert report_generator.model.invoke.called or hasattr(
|
|
report_generator.model, "invoke"
|
|
)
|
|
|
|
|
|
def test_research_and_generate_sections(report_generator):
|
|
"""Test researching and generating sections."""
|
|
# Define sample structure
|
|
structure = [
|
|
{
|
|
"name": "Introduction",
|
|
"subsections": [
|
|
{"name": "Background", "purpose": "Historical context"}
|
|
],
|
|
},
|
|
{
|
|
"name": "Findings",
|
|
"subsections": [
|
|
{"name": "Key Results", "purpose": "Main research outcomes"}
|
|
],
|
|
},
|
|
]
|
|
|
|
# Mock the search system to return specific results for each subsection
|
|
report_generator.search_system.analyze_topic.side_effect = [
|
|
{
|
|
"current_knowledge": "Background section content about historical context."
|
|
},
|
|
{
|
|
"current_knowledge": "Key results section content with main findings."
|
|
},
|
|
]
|
|
|
|
# Call the method
|
|
sections = report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial findings"}, structure, "Research query"
|
|
)
|
|
|
|
# Verify sections were generated correctly
|
|
assert "Introduction" in sections
|
|
assert "Findings" in sections
|
|
assert "# 1. Introduction" in sections["Introduction"]
|
|
assert "Background section content" in sections["Introduction"]
|
|
|
|
assert "# 2. Findings" in sections["Findings"]
|
|
assert "Key results section content" in sections["Findings"]
|
|
|
|
# Verify search system was called the correct number of times (once per subsection)
|
|
assert report_generator.search_system.analyze_topic.call_count == 2
|
|
|
|
|
|
def test_format_final_report(report_generator, monkeypatch):
|
|
"""Test formatting the final report."""
|
|
# Define sample structure and sections
|
|
structure = [
|
|
{
|
|
"name": "Introduction",
|
|
"subsections": [
|
|
{"name": "Background", "purpose": "Historical context"}
|
|
],
|
|
},
|
|
{
|
|
"name": "Findings",
|
|
"subsections": [
|
|
{"name": "Key Results", "purpose": "Main research outcomes"}
|
|
],
|
|
},
|
|
]
|
|
|
|
sections = {
|
|
"Introduction": "# Introduction\n\n## Background\n\nBackground content here.",
|
|
"Findings": "# Findings\n\n## Key Results\n\nKey results content here.",
|
|
}
|
|
|
|
# Mock format_links_to_markdown
|
|
def mock_format_links(all_links):
|
|
return "1. [Source 1](https://example.com/1)\n2. [Source 2](https://example.com/2)"
|
|
|
|
monkeypatch.setattr(
|
|
"local_deep_research.utilities.search_utilities.format_links_to_markdown",
|
|
mock_format_links,
|
|
)
|
|
|
|
# Call the method
|
|
report = report_generator._format_final_report(
|
|
sections, structure, "Test query"
|
|
)
|
|
|
|
# Verify report structure
|
|
assert "content" in report
|
|
assert "metadata" in report
|
|
assert "# Table of Contents" in report["content"]
|
|
assert "Introduction" in report["content"]
|
|
assert "Findings" in report["content"]
|
|
assert "Background content here" in report["content"]
|
|
assert "Key results content here" in report["content"]
|
|
assert "## Sources" in report["content"]
|
|
|
|
# Verify metadata
|
|
assert report["metadata"]["query"] == "Test query"
|
|
assert "generated_at" in report["metadata"]
|
|
assert "sections_researched" in report["metadata"]
|
|
assert report["metadata"]["sections_researched"] == 2
|
|
|
|
|
|
def test_generate_report(report_generator, sample_findings, monkeypatch):
|
|
"""Test the full report generation process."""
|
|
|
|
# Mock the component methods with Mock objects
|
|
mock_determine_structure = Mock(
|
|
return_value=[
|
|
{
|
|
"name": "Section",
|
|
"subsections": [{"name": "Subsection", "purpose": "Purpose"}],
|
|
}
|
|
]
|
|
)
|
|
|
|
mock_research = Mock(return_value={"Section": "Section content"})
|
|
|
|
mock_format = Mock(
|
|
return_value={
|
|
"content": "Report content",
|
|
"metadata": {"query": "Test query"},
|
|
}
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
report_generator,
|
|
"_determine_report_structure",
|
|
mock_determine_structure,
|
|
)
|
|
monkeypatch.setattr(
|
|
report_generator, "_research_and_generate_sections", mock_research
|
|
)
|
|
monkeypatch.setattr(report_generator, "_format_final_report", mock_format)
|
|
|
|
# Call generate_report
|
|
result = report_generator.generate_report(sample_findings, "Test query")
|
|
|
|
# Verify component methods were called with correct arguments
|
|
mock_determine_structure.assert_called_once_with(
|
|
sample_findings, "Test query"
|
|
)
|
|
|
|
# Get the expected structure result
|
|
structure_result = mock_determine_structure.return_value
|
|
mock_research.assert_called_once_with(
|
|
sample_findings, structure_result, "Test query", progress_callback=None
|
|
)
|
|
|
|
# Get the expected sections result
|
|
sections_result = mock_research.return_value
|
|
mock_format.assert_called_once_with(
|
|
sections_result, structure_result, "Test query"
|
|
)
|
|
|
|
# Verify result is the formatted report
|
|
assert result == mock_format.return_value
|
|
|
|
|
|
def test_generate_error_report(report_generator):
|
|
"""Test generating an error report."""
|
|
error_report = report_generator._generate_error_report(
|
|
"Test query", "Error message"
|
|
)
|
|
|
|
assert "Test query" in error_report
|
|
assert "Error message" in error_report
|
|
|
|
|
|
def test_context_accumulation_across_sections(report_generator):
|
|
"""Test that previous section content is passed to subsequent sections."""
|
|
# Define a structure with multiple sections
|
|
structure = [
|
|
{
|
|
"name": "Section A",
|
|
"subsections": [{"name": "Part 1", "purpose": "First part"}],
|
|
},
|
|
{
|
|
"name": "Section B",
|
|
"subsections": [{"name": "Part 2", "purpose": "Second part"}],
|
|
},
|
|
{
|
|
"name": "Section C",
|
|
"subsections": [{"name": "Part 3", "purpose": "Third part"}],
|
|
},
|
|
]
|
|
|
|
# Track the queries passed to analyze_topic
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
return {"current_knowledge": f"Content for query about {query[:50]}"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
# Generate sections
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial findings"}, structure, "Test query"
|
|
)
|
|
|
|
# Verify that analyze_topic was called 3 times
|
|
assert len(captured_queries) == 3
|
|
|
|
# First section should NOT have previous context
|
|
assert "CONTENT ALREADY WRITTEN" not in captured_queries[0]
|
|
|
|
# Second section SHOULD have previous context from first section
|
|
assert "CONTENT ALREADY WRITTEN" in captured_queries[1]
|
|
assert "Section A" in captured_queries[1]
|
|
|
|
# Third section SHOULD have previous context from first and second sections
|
|
assert "CONTENT ALREADY WRITTEN" in captured_queries[2]
|
|
assert "Section A" in captured_queries[2]
|
|
assert "Section B" in captured_queries[2]
|
|
|
|
|
|
def test_context_accumulation_limits_to_last_3_sections(report_generator):
|
|
"""Test that only the last 3 sections are included in context."""
|
|
# Define a structure with 5 sections
|
|
structure = [
|
|
{
|
|
"name": f"Section {i}",
|
|
"subsections": [{"name": f"Part {i}", "purpose": f"Purpose {i}"}],
|
|
}
|
|
for i in range(1, 6)
|
|
]
|
|
|
|
# Track the queries
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
return {
|
|
"current_knowledge": f"Content for section {len(captured_queries)}"
|
|
}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
# Generate sections
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial findings"}, structure, "Test query"
|
|
)
|
|
|
|
# The 5th section should only have context from sections 2, 3, 4 (last 3)
|
|
# Section 1 content should NOT be in the 5th query
|
|
fifth_query = captured_queries[4]
|
|
|
|
# Count how many section references are in the context
|
|
# Sections 2, 3, 4 should be present, Section 1 should not
|
|
assert "Section 2" in fifth_query
|
|
assert "Section 3" in fifth_query
|
|
assert "Section 4" in fifth_query
|
|
# Section 1 should have been dropped (only last 3 kept)
|
|
# Check that the context section exists but Section 1 is not in it
|
|
assert "CONTENT ALREADY WRITTEN" in fifth_query
|
|
|
|
|
|
def test_context_truncation_for_large_content(report_generator):
|
|
"""Test that context is truncated when it exceeds 4000 characters."""
|
|
# Define structure with sections that will generate large content
|
|
structure = [
|
|
{
|
|
"name": "Section A",
|
|
"subsections": [
|
|
{"name": "Large Part", "purpose": "Generate large content"}
|
|
],
|
|
},
|
|
{
|
|
"name": "Section B",
|
|
"subsections": [
|
|
{"name": "Next Part", "purpose": "Should see truncated context"}
|
|
],
|
|
},
|
|
]
|
|
|
|
# Track queries
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
# Return large content for first section (over 4000 chars)
|
|
if len(captured_queries) == 1:
|
|
return {"current_knowledge": "X" * 5000} # 5000 chars
|
|
return {"current_knowledge": "Normal content"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
# Generate sections
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial findings"}, structure, "Test query"
|
|
)
|
|
|
|
# Second query should have truncated context
|
|
second_query = captured_queries[1]
|
|
assert "CONTENT ALREADY WRITTEN" in second_query
|
|
assert "[...truncated]" in second_query
|
|
|
|
|
|
def test_context_includes_section_labels(report_generator):
|
|
"""Test that accumulated content includes section/subsection labels."""
|
|
structure = [
|
|
{
|
|
"name": "Introduction",
|
|
"subsections": [
|
|
{"name": "Overview", "purpose": "Provide overview"}
|
|
],
|
|
},
|
|
{
|
|
"name": "Details",
|
|
"subsections": [
|
|
{"name": "Specifics", "purpose": "Provide details"}
|
|
],
|
|
},
|
|
]
|
|
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
return {"current_knowledge": "Some content here"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# Second query should have labeled content from first section
|
|
second_query = captured_queries[1]
|
|
assert "[Introduction > Overview]" in second_query
|
|
|
|
|
|
def test_no_context_for_first_section(report_generator):
|
|
"""Test that the first section has no previous context."""
|
|
structure = [
|
|
{
|
|
"name": "First Section",
|
|
"subsections": [{"name": "First Part", "purpose": "First purpose"}],
|
|
},
|
|
]
|
|
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
return {"current_knowledge": "Content"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# First (and only) query should not have previous context markers
|
|
assert "CONTENT ALREADY WRITTEN" not in captured_queries[0]
|
|
assert "DO NOT REPEAT" not in captured_queries[0]
|
|
|
|
|
|
def test_context_accumulation_with_multiple_subsections(report_generator):
|
|
"""Test that context accumulates correctly across multiple subsections within a section."""
|
|
structure = [
|
|
{
|
|
"name": "Main Section",
|
|
"subsections": [
|
|
{"name": "Sub A", "purpose": "First subsection"},
|
|
{"name": "Sub B", "purpose": "Second subsection"},
|
|
{"name": "Sub C", "purpose": "Third subsection"},
|
|
],
|
|
},
|
|
]
|
|
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
return {
|
|
"current_knowledge": f"Content from subsection {len(captured_queries)}"
|
|
}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# Should have 3 queries (one per subsection)
|
|
assert len(captured_queries) == 3
|
|
|
|
# First subsection: no context
|
|
assert "CONTENT ALREADY WRITTEN" not in captured_queries[0]
|
|
|
|
# Second subsection: should have Sub A content
|
|
assert "CONTENT ALREADY WRITTEN" in captured_queries[1]
|
|
assert "Sub A" in captured_queries[1]
|
|
|
|
# Third subsection: should have Sub A and Sub B content
|
|
assert "CONTENT ALREADY WRITTEN" in captured_queries[2]
|
|
assert "Sub A" in captured_queries[2]
|
|
assert "Sub B" in captured_queries[2]
|
|
|
|
|
|
def test_context_includes_critical_instruction(report_generator):
|
|
"""Test that the CRITICAL instruction is included in prompts with context."""
|
|
structure = [
|
|
{
|
|
"name": "Section 1",
|
|
"subsections": [{"name": "Part 1", "purpose": "Purpose 1"}],
|
|
},
|
|
{
|
|
"name": "Section 2",
|
|
"subsections": [{"name": "Part 2", "purpose": "Purpose 2"}],
|
|
},
|
|
]
|
|
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
return {"current_knowledge": "Some content"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# Second query should have the CRITICAL instruction
|
|
second_query = captured_queries[1]
|
|
assert "CRITICAL" in second_query
|
|
assert "Do NOT repeat" in second_query
|
|
assert "Focus on NEW information" in second_query
|
|
|
|
|
|
def test_context_with_empty_subsection_content(report_generator):
|
|
"""Test that empty subsection results don't break context accumulation."""
|
|
structure = [
|
|
{
|
|
"name": "Section 1",
|
|
"subsections": [
|
|
{"name": "Empty Part", "purpose": "Returns nothing"}
|
|
],
|
|
},
|
|
{
|
|
"name": "Section 2",
|
|
"subsections": [
|
|
{"name": "Normal Part", "purpose": "Returns content"}
|
|
],
|
|
},
|
|
{
|
|
"name": "Section 3",
|
|
"subsections": [{"name": "Final Part", "purpose": "Should work"}],
|
|
},
|
|
]
|
|
|
|
call_count = [0]
|
|
|
|
def capture_query(query):
|
|
call_count[0] += 1
|
|
if call_count[0] == 1:
|
|
# First subsection returns no content
|
|
return {"current_knowledge": None}
|
|
return {"current_knowledge": f"Content {call_count[0]}"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
# Should not raise an error
|
|
sections = report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# Sections should still be generated
|
|
assert "Section 1" in sections
|
|
assert "Section 2" in sections
|
|
assert "Section 3" in sections
|
|
|
|
|
|
def test_context_format_has_clear_delimiters(report_generator):
|
|
"""Test that context block has clear start and end delimiters."""
|
|
structure = [
|
|
{
|
|
"name": "First",
|
|
"subsections": [{"name": "A", "purpose": "First"}],
|
|
},
|
|
{
|
|
"name": "Second",
|
|
"subsections": [{"name": "B", "purpose": "Second"}],
|
|
},
|
|
]
|
|
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
return {"current_knowledge": "Content here"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
second_query = captured_queries[1]
|
|
|
|
# Check for clear delimiters
|
|
assert "=== CONTENT ALREADY WRITTEN (DO NOT REPEAT) ===" in second_query
|
|
assert "=== END OF PREVIOUS CONTENT ===" in second_query
|
|
|
|
# Verify delimiters appear in correct order
|
|
start_pos = second_query.find("=== CONTENT ALREADY WRITTEN")
|
|
end_pos = second_query.find("=== END OF PREVIOUS CONTENT")
|
|
assert start_pos < end_pos
|
|
|
|
|
|
def test_section_level_vs_subsection_level_prompts(report_generator):
|
|
"""Test that both section-level (single subsection) and subsection-level prompts get context."""
|
|
structure = [
|
|
{
|
|
"name": "Standalone Section",
|
|
"subsections": [
|
|
{"name": "Only Part", "purpose": "Single subsection"}
|
|
],
|
|
},
|
|
{
|
|
"name": "Multi Section",
|
|
"subsections": [
|
|
{"name": "Part A", "purpose": "First of multiple"},
|
|
{"name": "Part B", "purpose": "Second of multiple"},
|
|
],
|
|
},
|
|
]
|
|
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
return {"current_knowledge": f"Content {len(captured_queries)}"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# Query 0: Standalone section (section-level, no context)
|
|
assert "CONTENT ALREADY WRITTEN" not in captured_queries[0]
|
|
|
|
# Query 1: First part of multi-section (subsection-level, has context)
|
|
assert "CONTENT ALREADY WRITTEN" in captured_queries[1]
|
|
assert "Standalone Section" in captured_queries[1]
|
|
|
|
# Query 2: Second part of multi-section (subsection-level, has context)
|
|
assert "CONTENT ALREADY WRITTEN" in captured_queries[2]
|
|
|
|
|
|
def test_context_preserves_content_integrity(report_generator):
|
|
"""Test that the actual content is preserved in context, not just labels."""
|
|
structure = [
|
|
{
|
|
"name": "Section A",
|
|
"subsections": [{"name": "Part 1", "purpose": "First"}],
|
|
},
|
|
{
|
|
"name": "Section B",
|
|
"subsections": [{"name": "Part 2", "purpose": "Second"}],
|
|
},
|
|
]
|
|
|
|
unique_content = "UNIQUE_MARKER_12345_THIS_SHOULD_APPEAR_IN_CONTEXT"
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
if len(captured_queries) == 1:
|
|
return {"current_knowledge": unique_content}
|
|
return {"current_knowledge": "Other content"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# The unique content from section 1 should appear in section 2's query
|
|
assert unique_content in captured_queries[1]
|
|
|
|
|
|
def test_context_with_special_characters(report_generator):
|
|
"""Test that content with special characters doesn't break context."""
|
|
structure = [
|
|
{
|
|
"name": "Section 1",
|
|
"subsections": [{"name": "Part 1", "purpose": "First"}],
|
|
},
|
|
{
|
|
"name": "Section 2",
|
|
"subsections": [{"name": "Part 2", "purpose": "Second"}],
|
|
},
|
|
]
|
|
|
|
special_content = "Content with 'quotes', \"double quotes\", {braces}, [brackets], and $pecial ch@rs!"
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
if len(captured_queries) == 1:
|
|
return {"current_knowledge": special_content}
|
|
return {"current_knowledge": "Normal"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
# Should not raise any errors
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# Special content should be in second query's context
|
|
assert special_content in captured_queries[1]
|
|
|
|
|
|
def test_accumulated_findings_count_matches_subsections(report_generator):
|
|
"""Test that the number of accumulated findings matches successful subsections."""
|
|
structure = [
|
|
{
|
|
"name": "Section 1",
|
|
"subsections": [
|
|
{"name": "Sub 1", "purpose": "P1"},
|
|
{"name": "Sub 2", "purpose": "P2"},
|
|
],
|
|
},
|
|
{
|
|
"name": "Section 2",
|
|
"subsections": [
|
|
{"name": "Sub 3", "purpose": "P3"},
|
|
],
|
|
},
|
|
]
|
|
|
|
call_count = [0]
|
|
|
|
def capture_query(query):
|
|
call_count[0] += 1
|
|
return {"current_knowledge": f"Content {call_count[0]}"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# Should have called analyze_topic 3 times (once per subsection)
|
|
assert call_count[0] == 3
|
|
|
|
|
|
def test_context_separator_between_sections(report_generator):
|
|
"""Test that sections in context are separated by the correct delimiter."""
|
|
structure = [
|
|
{
|
|
"name": f"Section {i}",
|
|
"subsections": [{"name": f"Part {i}", "purpose": f"P{i}"}],
|
|
}
|
|
for i in range(1, 4)
|
|
]
|
|
|
|
captured_queries = []
|
|
|
|
def capture_query(query):
|
|
captured_queries.append(query)
|
|
return {"current_knowledge": f"Content {len(captured_queries)}"}
|
|
|
|
report_generator.search_system.analyze_topic.side_effect = capture_query
|
|
|
|
report_generator._research_and_generate_sections(
|
|
{"current_knowledge": "Initial"}, structure, "Query"
|
|
)
|
|
|
|
# Third query should have separator between Section 1 and Section 2 content
|
|
third_query = captured_queries[2]
|
|
assert "---" in third_query # The separator used between sections
|
|
|
|
|
|
def test_truncate_at_sentence_boundary_no_truncation(report_generator):
|
|
"""Test that short text is not truncated."""
|
|
text = "This is short. It should not be truncated."
|
|
result = report_generator._truncate_at_sentence_boundary(text, 1000)
|
|
assert result == text
|
|
assert "[...truncated]" not in result
|
|
|
|
|
|
def test_truncate_at_sentence_boundary_at_sentence(report_generator):
|
|
"""Test that truncation happens at sentence boundary when possible."""
|
|
# Use a longer limit so sentence boundary falls within 80% threshold
|
|
text = "First sentence. Second sentence. Third sentence that is very long and goes beyond the limit here."
|
|
result = report_generator._truncate_at_sentence_boundary(text, 40)
|
|
# "First sentence. Second sentence." = 33 chars, which is > 80% of 40 (32)
|
|
# So it should truncate at the sentence boundary
|
|
assert result.startswith("First sentence. Second sentence.")
|
|
assert "[...truncated]" in result
|
|
assert "Third sentence" not in result
|
|
|
|
|
|
def test_truncate_at_sentence_boundary_fallback(report_generator):
|
|
"""Test that truncation falls back to hard cut when no good boundary exists."""
|
|
# No sentence boundaries in the first 80% of the limit
|
|
text = "A" * 100 # No sentence boundaries
|
|
result = report_generator._truncate_at_sentence_boundary(text, 50)
|
|
# Should hard truncate at 50 chars
|
|
assert len(result) == 50 + len("\n[...truncated]")
|
|
assert result.startswith("A" * 50)
|
|
assert "[...truncated]" in result
|
|
|
|
|
|
def test_truncate_at_sentence_boundary_with_question_mark(report_generator):
|
|
"""Test that question marks are recognized as sentence boundaries."""
|
|
text = "Is this a question? Yes it is. More content that exceeds the limit."
|
|
result = report_generator._truncate_at_sentence_boundary(text, 35)
|
|
assert "Is this a question?" in result
|
|
assert "[...truncated]" in result
|
|
|
|
|
|
def test_truncate_at_sentence_boundary_with_exclamation(report_generator):
|
|
"""Test that exclamation marks are recognized as sentence boundaries."""
|
|
text = "Wow! Amazing! This is great content that exceeds the limit significantly."
|
|
result = report_generator._truncate_at_sentence_boundary(text, 20)
|
|
assert "Amazing!" in result or "Wow!" in result
|
|
assert "[...truncated]" in result
|
|
|
|
|
|
def test_build_previous_context_empty(report_generator):
|
|
"""Test that empty accumulated findings returns empty string."""
|
|
result = report_generator._build_previous_context([])
|
|
assert result == ""
|
|
|
|
|
|
def test_build_previous_context_single_section(report_generator):
|
|
"""Test context building with a single section."""
|
|
accumulated = ["[Section A > Part 1]\nContent for section A"]
|
|
result = report_generator._build_previous_context(accumulated)
|
|
assert "CONTENT ALREADY WRITTEN" in result
|
|
assert "Section A" in result
|
|
assert "CRITICAL" in result
|
|
assert "END OF PREVIOUS CONTENT" in result
|
|
|
|
|
|
def test_build_previous_context_respects_max_sections(report_generator):
|
|
"""Test that only max_context_sections sections are included."""
|
|
# Use the instance's max_context_sections (defaults to DEFAULT_MAX_CONTEXT_SECTIONS)
|
|
max_sections = report_generator.max_context_sections
|
|
|
|
# Create more sections than the limit
|
|
accumulated = [f"[Section {i}]\nContent {i}" for i in range(10)]
|
|
result = report_generator._build_previous_context(accumulated)
|
|
|
|
# Should only include the last max_sections
|
|
for i in range(10 - max_sections):
|
|
# Earlier sections should NOT be in the context
|
|
if f"Section {i}" in result:
|
|
# Check it's not just a coincidental match
|
|
assert f"[Section {i}]" not in result or i >= 10 - max_sections
|
|
|
|
# Last sections SHOULD be present
|
|
assert f"Section {10 - 1}" in result # Last section
|
|
|
|
|
|
def test_build_previous_context_truncates_long_content(report_generator):
|
|
"""Test that very long content is truncated."""
|
|
|
|
# Create content that exceeds max_context_chars (default 4000)
|
|
long_content = "This is a sentence. " * 500 # ~10000 chars
|
|
accumulated = [f"[Section A]\n{long_content}"]
|
|
result = report_generator._build_previous_context(accumulated)
|
|
|
|
# Result should contain truncation marker
|
|
assert "[...truncated]" in result
|
|
# Total context (excluding delimiters) should be around max_context_chars
|
|
# The result includes delimiters so it will be larger
|
|
|
|
|
|
def test_configurable_max_context_sections(
|
|
mock_llm, mock_search_system, monkeypatch
|
|
):
|
|
"""Test that max_context_sections can be configured via settings_snapshot."""
|
|
monkeypatch.setattr(
|
|
"local_deep_research.report_generator.get_llm", lambda: mock_llm
|
|
)
|
|
|
|
# Create generator with custom settings
|
|
settings_snapshot = {
|
|
"report.max_context_sections": 5,
|
|
"report.max_context_chars": 8000,
|
|
}
|
|
generator = IntegratedReportGenerator(
|
|
search_system=mock_search_system,
|
|
settings_snapshot=settings_snapshot,
|
|
)
|
|
|
|
# Verify settings were applied
|
|
assert generator.max_context_sections == 5
|
|
assert generator.max_context_chars == 8000
|
|
|
|
|
|
def test_configurable_max_context_sections_affects_context_building(
|
|
mock_llm, mock_search_system, monkeypatch
|
|
):
|
|
"""Test that custom max_context_sections affects _build_previous_context."""
|
|
monkeypatch.setattr(
|
|
"local_deep_research.report_generator.get_llm", lambda: mock_llm
|
|
)
|
|
|
|
# Create generator with only 2 sections in context
|
|
settings_snapshot = {
|
|
"report.max_context_sections": 2,
|
|
}
|
|
generator = IntegratedReportGenerator(
|
|
search_system=mock_search_system,
|
|
settings_snapshot=settings_snapshot,
|
|
)
|
|
|
|
# Create 5 sections
|
|
accumulated = [f"[Section {i}]\nContent {i}" for i in range(5)]
|
|
result = generator._build_previous_context(accumulated)
|
|
|
|
# Should only include Section 3 and Section 4 (last 2)
|
|
assert "[Section 3]" in result
|
|
assert "[Section 4]" in result
|
|
# Section 0, 1, 2 should NOT be in context (only last 2 kept)
|
|
assert "[Section 0]" not in result
|
|
assert "[Section 1]" not in result
|
|
assert "[Section 2]" not in result
|
|
|
|
|
|
def test_configurable_max_context_chars_affects_truncation(
|
|
mock_llm, mock_search_system, monkeypatch
|
|
):
|
|
"""Test that custom max_context_chars affects truncation behavior."""
|
|
monkeypatch.setattr(
|
|
"local_deep_research.report_generator.get_llm", lambda: mock_llm
|
|
)
|
|
|
|
# Create generator with small context limit
|
|
settings_snapshot = {
|
|
"report.max_context_chars": 100,
|
|
}
|
|
generator = IntegratedReportGenerator(
|
|
search_system=mock_search_system,
|
|
settings_snapshot=settings_snapshot,
|
|
)
|
|
|
|
# Create content that exceeds 100 chars but is under default 4000
|
|
content = "This is a sentence. " * 20 # ~400 chars
|
|
accumulated = [f"[Section A]\n{content}"]
|
|
result = generator._build_previous_context(accumulated)
|
|
|
|
# Should be truncated due to small limit
|
|
assert "[...truncated]" in result
|
|
|
|
|
|
def test_default_context_settings_when_no_snapshot(
|
|
mock_llm, mock_search_system, monkeypatch
|
|
):
|
|
"""Test that default values are used when no settings_snapshot is provided."""
|
|
from local_deep_research.report_generator import (
|
|
DEFAULT_MAX_CONTEXT_SECTIONS,
|
|
DEFAULT_MAX_CONTEXT_CHARS,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
"local_deep_research.report_generator.get_llm", lambda: mock_llm
|
|
)
|
|
|
|
generator = IntegratedReportGenerator(search_system=mock_search_system)
|
|
|
|
# Should use defaults
|
|
assert generator.max_context_sections == DEFAULT_MAX_CONTEXT_SECTIONS
|
|
assert generator.max_context_chars == DEFAULT_MAX_CONTEXT_CHARS
|