wehub-resource-sync
222 lines
7.5 KiB
Python
222 lines
7.5 KiB
Python
import importlib.util
|
|
import sys
|
|
import types
|
|
from pathlib import Path
|
|
|
|
|
|
class FakeParser:
|
|
OFFICE_FORMATS = {".docx"}
|
|
IMAGE_FORMATS = {".png"}
|
|
TEXT_FORMATS = {".txt", ".md"}
|
|
|
|
def __init__(self):
|
|
self.processed_files = []
|
|
|
|
def check_installation(self):
|
|
return True
|
|
|
|
def parse_document(self, file_path, output_dir, method="auto", **kwargs):
|
|
self.processed_files.append(file_path)
|
|
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
|
return [{"type": "text", "text": Path(file_path).read_text()}]
|
|
|
|
|
|
def _make_batch_parser(monkeypatch):
|
|
fake_parser = FakeParser()
|
|
repo_root = Path(__file__).parents[1]
|
|
|
|
package = types.ModuleType("raganything")
|
|
package.__path__ = [str(repo_root / "raganything")]
|
|
parser_module = types.ModuleType("raganything.parser")
|
|
parser_module.get_parser = lambda parser_type: fake_parser
|
|
|
|
monkeypatch.setitem(sys.modules, "raganything", package)
|
|
monkeypatch.setitem(sys.modules, "raganything.parser", parser_module)
|
|
sys.modules.pop("raganything.batch_parser", None)
|
|
|
|
spec = importlib.util.spec_from_file_location(
|
|
"raganything.batch_parser",
|
|
repo_root / "raganything" / "batch_parser.py",
|
|
)
|
|
batch_parser_module = importlib.util.module_from_spec(spec)
|
|
monkeypatch.setitem(sys.modules, "raganything.batch_parser", batch_parser_module)
|
|
spec.loader.exec_module(batch_parser_module)
|
|
|
|
batch_parser = batch_parser_module.BatchParser(
|
|
parser_type="fake",
|
|
max_workers=1,
|
|
show_progress=False,
|
|
skip_installation_check=True,
|
|
)
|
|
return batch_parser, fake_parser
|
|
|
|
|
|
def test_incremental_batch_skips_unchanged_files(monkeypatch, tmp_path):
|
|
batch_parser, fake_parser = _make_batch_parser(monkeypatch)
|
|
docs_dir = tmp_path / "docs"
|
|
docs_dir.mkdir()
|
|
first_doc = docs_dir / "a.txt"
|
|
second_doc = docs_dir / "b.txt"
|
|
first_doc.write_text("alpha", encoding="utf-8")
|
|
second_doc.write_text("beta", encoding="utf-8")
|
|
output_dir = tmp_path / "out"
|
|
|
|
first_result = batch_parser.process_batch(
|
|
[str(docs_dir)],
|
|
str(output_dir),
|
|
incremental=True,
|
|
)
|
|
|
|
assert set(first_result.successful_files) == {str(first_doc), str(second_doc)}
|
|
assert first_result.skipped_files == []
|
|
assert set(fake_parser.processed_files) == {str(first_doc), str(second_doc)}
|
|
|
|
fake_parser.processed_files.clear()
|
|
second_result = batch_parser.process_batch(
|
|
[str(docs_dir)],
|
|
str(output_dir),
|
|
incremental=True,
|
|
)
|
|
|
|
assert second_result.successful_files == []
|
|
assert set(second_result.skipped_files) == {str(first_doc), str(second_doc)}
|
|
assert second_result.success_rate == 100.0
|
|
assert fake_parser.processed_files == []
|
|
|
|
first_doc.write_text("alpha changed", encoding="utf-8")
|
|
third_result = batch_parser.process_batch(
|
|
[str(docs_dir)],
|
|
str(output_dir),
|
|
incremental=True,
|
|
)
|
|
|
|
assert third_result.successful_files == [str(first_doc)]
|
|
assert third_result.skipped_files == [str(second_doc)]
|
|
assert fake_parser.processed_files == [str(first_doc)]
|
|
|
|
|
|
def test_incremental_dry_run_reports_changed_and_skipped_files(monkeypatch, tmp_path):
|
|
batch_parser, fake_parser = _make_batch_parser(monkeypatch)
|
|
docs_dir = tmp_path / "docs"
|
|
docs_dir.mkdir()
|
|
first_doc = docs_dir / "a.txt"
|
|
second_doc = docs_dir / "b.txt"
|
|
first_doc.write_text("alpha", encoding="utf-8")
|
|
second_doc.write_text("beta", encoding="utf-8")
|
|
output_dir = tmp_path / "out"
|
|
|
|
batch_parser.process_batch([str(docs_dir)], str(output_dir), incremental=True)
|
|
fake_parser.processed_files.clear()
|
|
|
|
first_doc.write_text("alpha changed", encoding="utf-8")
|
|
dry_run_result = batch_parser.process_batch(
|
|
[str(docs_dir)],
|
|
str(output_dir),
|
|
incremental=True,
|
|
dry_run=True,
|
|
)
|
|
|
|
assert dry_run_result.dry_run is True
|
|
assert dry_run_result.successful_files == [str(first_doc)]
|
|
assert dry_run_result.skipped_files == [str(second_doc)]
|
|
assert fake_parser.processed_files == []
|
|
|
|
|
|
def _seed_two_docs(tmp_path):
|
|
docs_dir = tmp_path / "docs"
|
|
docs_dir.mkdir()
|
|
first_doc = docs_dir / "a.txt"
|
|
second_doc = docs_dir / "b.txt"
|
|
first_doc.write_text("alpha", encoding="utf-8")
|
|
second_doc.write_text("beta", encoding="utf-8")
|
|
return docs_dir, first_doc, second_doc
|
|
|
|
|
|
def test_incremental_skip_does_not_rehash_unchanged_files(monkeypatch, tmp_path):
|
|
batch_parser, _ = _make_batch_parser(monkeypatch)
|
|
docs_dir, first_doc, second_doc = _seed_two_docs(tmp_path)
|
|
output_dir = tmp_path / "out"
|
|
|
|
batch_parser.process_batch([str(docs_dir)], str(output_dir), incremental=True)
|
|
|
|
# Second run: both files are unchanged, so the size+mtime fast path must
|
|
# skip them without hashing a single byte.
|
|
hashed = []
|
|
real_md5 = type(batch_parser)._compute_md5
|
|
|
|
def counting_md5(file_path):
|
|
hashed.append(file_path)
|
|
return real_md5(file_path)
|
|
|
|
monkeypatch.setattr(type(batch_parser), "_compute_md5", staticmethod(counting_md5))
|
|
|
|
result = batch_parser.process_batch(
|
|
[str(docs_dir)], str(output_dir), incremental=True
|
|
)
|
|
|
|
assert set(result.skipped_files) == {str(first_doc), str(second_doc)}
|
|
assert hashed == []
|
|
|
|
|
|
def test_incremental_ignores_corrupt_manifest(monkeypatch, tmp_path):
|
|
batch_parser, fake_parser = _make_batch_parser(monkeypatch)
|
|
docs_dir, first_doc, second_doc = _seed_two_docs(tmp_path)
|
|
output_dir = tmp_path / "out"
|
|
|
|
batch_parser.process_batch([str(docs_dir)], str(output_dir), incremental=True)
|
|
|
|
manifest_path = output_dir / ".raganything_batch_manifest.json"
|
|
manifest_path.write_text("this is not valid json{", encoding="utf-8")
|
|
fake_parser.processed_files.clear()
|
|
|
|
result = batch_parser.process_batch(
|
|
[str(docs_dir)], str(output_dir), incremental=True
|
|
)
|
|
|
|
# A corrupt manifest must be ignored and every file reprocessed.
|
|
assert set(result.successful_files) == {str(first_doc), str(second_doc)}
|
|
assert result.skipped_files == []
|
|
|
|
|
|
def test_incremental_tolerates_unreadable_file(monkeypatch, tmp_path):
|
|
batch_parser, fake_parser = _make_batch_parser(monkeypatch)
|
|
docs_dir, first_doc, second_doc = _seed_two_docs(tmp_path)
|
|
output_dir = tmp_path / "out"
|
|
|
|
batch_parser.process_batch([str(docs_dir)], str(output_dir), incremental=True)
|
|
fake_parser.processed_files.clear()
|
|
|
|
# Simulate one file becoming unreadable between discovery and the scan.
|
|
real_metadata = type(batch_parser)._file_metadata
|
|
|
|
def flaky_metadata(file_path):
|
|
if file_path == str(first_doc):
|
|
raise OSError("simulated unreadable file")
|
|
return real_metadata(file_path)
|
|
|
|
monkeypatch.setattr(
|
|
type(batch_parser), "_file_metadata", staticmethod(flaky_metadata)
|
|
)
|
|
|
|
# Must not raise; the unreadable file is treated as changed (queued for
|
|
# processing) while the other unchanged file is still skipped.
|
|
result = batch_parser.process_batch(
|
|
[str(docs_dir)], str(output_dir), incremental=True
|
|
)
|
|
|
|
assert str(first_doc) in (result.successful_files + result.failed_files)
|
|
assert result.skipped_files == [str(second_doc)]
|
|
|
|
|
|
def test_incremental_dry_run_does_not_write_manifest(monkeypatch, tmp_path):
|
|
batch_parser, _ = _make_batch_parser(monkeypatch)
|
|
docs_dir, _first_doc, _second_doc = _seed_two_docs(tmp_path)
|
|
output_dir = tmp_path / "out"
|
|
|
|
batch_parser.process_batch(
|
|
[str(docs_dir)], str(output_dir), incremental=True, dry_run=True
|
|
)
|
|
|
|
manifest_path = output_dir / ".raganything_batch_manifest.json"
|
|
assert not manifest_path.exists()
|