wehub-resource-sync
cddb07a176
docs / deploy (push) Has been cancelled
docs / changes (push) Has been cancelled
docs / check-and-build (push) Has been cancelled
build container image / cpu (push) Has been cancelled
build container image / cuda (push) Has been cancelled
build container image / rocm (push) Has been cancelled
frontend checks / frontend-checks (push) Has been cancelled
frontend tests / frontend-tests (push) Has been cancelled
lfs checks / lfs-check (push) Has been cancelled
python checks / python-checks (push) Has been cancelled
python tests / py3.12: macos-default (push) Has been cancelled
python tests / py3.11: windows-cpu (push) Has been cancelled
python tests / py3.12: windows-cpu (push) Has been cancelled
python tests / py3.11: linux-cpu (push) Has been cancelled
typegen checks / typegen-checks (push) Has been cancelled
uv lock checks / uv-lock-checks (push) Has been cancelled
openapi checks / openapi-checks (push) Has been cancelled
python tests / py3.11: macos-default (push) Has been cancelled
python tests / py3.12: linux-cpu (push) Has been cancelled
728 lines
28 KiB
Python
728 lines
28 KiB
Python
import os
|
|
import threading
|
|
from pathlib import Path
|
|
from shutil import copy2
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
from invokeai.app.services.config.config_default import InvokeAIAppConfig
|
|
from invokeai.app.services.image_files.image_files_disk import DiskImageFileStorage
|
|
from invokeai.app.services.image_moves.image_moves_default import (
|
|
ImageMoveQueueActive,
|
|
ImageMoveService,
|
|
)
|
|
from invokeai.app.services.image_records.image_records_common import ImageCategory, ResourceOrigin
|
|
from invokeai.app.services.image_records.image_records_sqlite import SqliteImageRecordStorage
|
|
from invokeai.app.services.session_queue.session_queue_common import DEFAULT_QUEUE_ID, SessionQueueStatus
|
|
from invokeai.app.services.shared.sqlite.sqlite_database import SqliteDatabase
|
|
from invokeai.app.services.shared.sqlite.sqlite_util import init_db
|
|
from invokeai.backend.util.logging import InvokeAILogger
|
|
|
|
|
|
def _build_db(tmp_path: Path) -> SqliteDatabase:
|
|
logger = InvokeAILogger.get_logger()
|
|
config = InvokeAIAppConfig(use_memory_db=False)
|
|
config._root = tmp_path
|
|
image_files = DiskImageFileStorage(tmp_path / "images")
|
|
return init_db(config=config, logger=logger, image_files=image_files)
|
|
|
|
|
|
def _save_record(
|
|
records: SqliteImageRecordStorage,
|
|
image_name: str,
|
|
subfolder: str,
|
|
created_at: str,
|
|
is_intermediate: bool = False,
|
|
) -> None:
|
|
records.save(
|
|
image_name=image_name,
|
|
image_origin=ResourceOrigin.INTERNAL,
|
|
image_category=ImageCategory.GENERAL,
|
|
width=16,
|
|
height=16,
|
|
has_workflow=False,
|
|
is_intermediate=is_intermediate,
|
|
image_subfolder=subfolder,
|
|
)
|
|
with records._db.transaction() as cursor:
|
|
cursor.execute("UPDATE images SET created_at = ? WHERE image_name = ?;", (created_at, image_name))
|
|
|
|
|
|
def _save_image(
|
|
service: ImageMoveService,
|
|
records: SqliteImageRecordStorage,
|
|
image_name: str,
|
|
subfolder: str,
|
|
created_at: str,
|
|
color: str,
|
|
is_intermediate: bool = False,
|
|
) -> None:
|
|
_save_record(
|
|
records,
|
|
image_name=image_name,
|
|
subfolder=subfolder,
|
|
created_at=created_at,
|
|
is_intermediate=is_intermediate,
|
|
)
|
|
service.image_files.save(Image.new("RGB", (16, 16), color), image_name=image_name, image_subfolder=subfolder)
|
|
|
|
|
|
def _service(tmp_path: Path, strategy: str = "date") -> tuple[ImageMoveService, SqliteImageRecordStorage]:
|
|
db = _build_db(tmp_path)
|
|
records = SqliteImageRecordStorage(db=db)
|
|
storage = DiskImageFileStorage(tmp_path / "images")
|
|
invoker = MagicMock()
|
|
invoker.services.configuration.pil_compress_level = 6
|
|
storage.start(invoker)
|
|
config = InvokeAIAppConfig(use_memory_db=True, image_subfolder_strategy=strategy)
|
|
config._root = tmp_path
|
|
service = ImageMoveService(db=db, image_files=storage, config=config, logger=InvokeAILogger.get_logger())
|
|
return service, records
|
|
|
|
|
|
def _job_item_states(service: ImageMoveService, job_id: int) -> dict[str, str]:
|
|
with service._db.transaction() as cursor:
|
|
cursor.execute(
|
|
"SELECT image_name, state FROM image_subfolder_move_items WHERE job_id = ? ORDER BY image_name;",
|
|
(job_id,),
|
|
)
|
|
return {row["image_name"]: row["state"] for row in cursor.fetchall()}
|
|
|
|
|
|
def _job_states(service: ImageMoveService) -> dict[int, str]:
|
|
with service._db.transaction() as cursor:
|
|
cursor.execute("SELECT id, state FROM image_subfolder_move_jobs ORDER BY id;")
|
|
return {row["id"]: row["state"] for row in cursor.fetchall()}
|
|
|
|
|
|
def test_move_all_images_uses_created_at_for_date_strategy(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-a.png"
|
|
_save_record(records, image_name=image_name, subfolder="", created_at="2024-02-03 04:05:06.000")
|
|
service.image_files.save(Image.new("RGB", (16, 16), "red"), image_name=image_name)
|
|
|
|
result = service.move_all_images()
|
|
|
|
assert result.planned == 1
|
|
assert result.committed == 1
|
|
record = records.get(image_name)
|
|
assert record.image_subfolder == "2024/02/03"
|
|
assert service.image_files.get_path(image_name, image_subfolder="2024/02/03").exists()
|
|
assert not service.image_files.get_path(image_name, image_subfolder="").exists()
|
|
|
|
|
|
def test_missing_intermediate_source_file_is_treated_as_success(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "missing-intermediate.png"
|
|
_save_record(
|
|
records,
|
|
image_name=image_name,
|
|
subfolder="",
|
|
created_at="2024-02-04 04:05:06.000",
|
|
is_intermediate=True,
|
|
)
|
|
|
|
result = service.move_all_images()
|
|
|
|
assert result.planned == 1
|
|
assert result.committed == 1
|
|
assert result.errors == 0
|
|
record = records.get(image_name)
|
|
assert record.image_subfolder == "2024/02/04"
|
|
assert service.get_latest_job().state == "committed"
|
|
|
|
|
|
def test_missing_intermediate_source_file_removes_orphaned_thumbnail(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "missing-intermediate-with-thumbnail.png"
|
|
old_subfolder = "old/intermediate"
|
|
_save_image(
|
|
service,
|
|
records,
|
|
image_name=image_name,
|
|
subfolder=old_subfolder,
|
|
created_at="2024-02-04 04:05:06.000",
|
|
color="red",
|
|
is_intermediate=True,
|
|
)
|
|
old_path = service.image_files.get_path(image_name, image_subfolder=old_subfolder)
|
|
old_thumbnail_path = service.image_files.get_path(image_name, thumbnail=True, image_subfolder=old_subfolder)
|
|
assert old_thumbnail_path.exists()
|
|
old_path.unlink()
|
|
|
|
result = service.move_all_images()
|
|
|
|
assert result.committed == 1
|
|
assert not old_thumbnail_path.exists()
|
|
assert records.get(image_name).image_subfolder == "2024/02/04"
|
|
|
|
|
|
def test_missing_non_intermediate_source_file_still_fails(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "missing-general.png"
|
|
_save_record(
|
|
records,
|
|
image_name=image_name,
|
|
subfolder="",
|
|
created_at="2024-02-04 04:05:06.000",
|
|
is_intermediate=False,
|
|
)
|
|
|
|
with pytest.raises(FileNotFoundError, match="Source image does not exist"):
|
|
service.plan_batch(last_image_name="", limit=100)
|
|
|
|
|
|
def test_move_all_images_continues_after_missing_non_intermediate_source_file(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
missing_image_name = "missing-general.png"
|
|
valid_image_name = "valid-general.png"
|
|
_save_record(
|
|
records,
|
|
image_name=missing_image_name,
|
|
subfolder="",
|
|
created_at="2024-02-04 04:05:06.000",
|
|
is_intermediate=False,
|
|
)
|
|
_save_image(service, records, valid_image_name, "", "2024-02-05 04:05:06.000", "blue")
|
|
|
|
result = service.move_all_images()
|
|
|
|
assert result.errors == 1
|
|
assert result.committed == 1
|
|
assert records.get(missing_image_name).image_subfolder == ""
|
|
assert records.get(valid_image_name).image_subfolder == "2024/02/05"
|
|
assert "error" in _job_states(service).values()
|
|
assert "committed" in _job_states(service).values()
|
|
|
|
|
|
def test_recovery_treats_missing_intermediate_source_file_as_success(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "missing-intermediate-recovery.png"
|
|
_save_record(
|
|
records,
|
|
image_name=image_name,
|
|
subfolder="",
|
|
created_at="2024-02-05 04:05:06.000",
|
|
is_intermediate=True,
|
|
)
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
job_id = service.create_move_job(moves)
|
|
|
|
recovered = service.startup_recovery()
|
|
|
|
assert recovered.committed == 1
|
|
assert recovered.errors == 0
|
|
assert records.get(image_name).image_subfolder == "2024/02/05"
|
|
assert service.get_job(job_id).state == "committed"
|
|
assert _job_item_states(service, job_id) == {image_name: "committed"}
|
|
|
|
|
|
def test_startup_recovery_commits_after_files_moved_but_db_not_updated(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-b.png"
|
|
_save_record(records, image_name=image_name, subfolder="", created_at="2025-06-07 08:09:10.000")
|
|
service.image_files.save(Image.new("RGB", (16, 16), "blue"), image_name=image_name)
|
|
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
job_id = service.create_move_job(moves)
|
|
service.perform_filesystem_moves(job_id)
|
|
|
|
assert records.get(image_name).image_subfolder == ""
|
|
|
|
recovered = service.startup_recovery()
|
|
|
|
assert recovered.committed == 1
|
|
assert records.get(image_name).image_subfolder == "2025/06/07"
|
|
assert service.get_job(job_id).state == "committed"
|
|
|
|
|
|
def test_status_reports_unplanned_images_after_recovery(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
_save_image(service, records, "image-recovered.png", "", "2024-02-03 04:05:06.000", "red")
|
|
_save_image(service, records, "image-unplanned.png", "", "2024-02-04 04:05:06.000", "blue")
|
|
moves = service.plan_batch(last_image_name="", limit=1)
|
|
job_id = service.create_move_job(moves)
|
|
service.perform_filesystem_moves(job_id)
|
|
|
|
recovered = service.startup_recovery()
|
|
status = service.get_background_status()
|
|
|
|
assert recovered.committed == 1
|
|
assert records.get("image-recovered.png").image_subfolder == "2024/02/03"
|
|
assert records.get("image-unplanned.png").image_subfolder == ""
|
|
assert status.active_job_id is None
|
|
assert status.latest_job is not None
|
|
assert status.latest_job.state == "committed"
|
|
assert status.needs_move_count == 1
|
|
|
|
|
|
def test_cleanup_empty_source_directories_after_move(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-c.png"
|
|
old_subfolder = "old/nested"
|
|
_save_record(records, image_name=image_name, subfolder=old_subfolder, created_at="2024-11-12 01:02:03.000")
|
|
service.image_files.save(Image.new("RGB", (16, 16), "green"), image_name=image_name, image_subfolder=old_subfolder)
|
|
old_parent = service.image_files.get_path(image_name, image_subfolder=old_subfolder).parent
|
|
old_thumb_parent = service.image_files.get_path(image_name, thumbnail=True, image_subfolder=old_subfolder).parent
|
|
|
|
service.move_all_images()
|
|
|
|
assert not old_parent.exists()
|
|
assert not old_thumb_parent.exists()
|
|
assert service.image_files.image_root.exists()
|
|
assert service.image_files.thumbnail_root.exists()
|
|
|
|
|
|
@pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are not supported on this platform")
|
|
def test_cleanup_empty_source_directories_stays_within_symlinked_root(tmp_path: Path) -> None:
|
|
service, _records = _service(tmp_path, strategy="date")
|
|
real_root = tmp_path / "real-root"
|
|
linked_root = tmp_path / "linked-root"
|
|
sibling = tmp_path / "sibling"
|
|
real_root.mkdir()
|
|
sibling.mkdir()
|
|
try:
|
|
linked_root.symlink_to(real_root, target_is_directory=True)
|
|
except OSError as e:
|
|
pytest.skip(f"symlink creation is not available: {e}")
|
|
nested = linked_root / "old" / "nested"
|
|
nested.mkdir(parents=True)
|
|
|
|
service._remove_empty_parents(nested, linked_root)
|
|
|
|
assert real_root.exists()
|
|
assert linked_root.exists()
|
|
assert sibling.exists()
|
|
assert not (real_root / "old").exists()
|
|
|
|
|
|
def test_startup_recovery_cleans_empty_source_directories(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-recovery-cleanup.png"
|
|
old_subfolder = "old/recovery"
|
|
_save_image(service, records, image_name, old_subfolder, "2024-11-13 01:02:03.000", "green")
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
job_id = service.create_move_job(moves)
|
|
move = moves[0]
|
|
old_parent = service.image_files.get_path(image_name, image_subfolder=old_subfolder).parent
|
|
old_thumb_parent = service.image_files.get_path(image_name, thumbnail=True, image_subfolder=old_subfolder).parent
|
|
move.new_path.parent.mkdir(parents=True, exist_ok=True)
|
|
move.new_thumbnail_path.parent.mkdir(parents=True, exist_ok=True)
|
|
move.old_path.replace(move.new_path)
|
|
move.old_thumbnail_path.replace(move.new_thumbnail_path)
|
|
|
|
recovered = service.startup_recovery()
|
|
|
|
assert recovered.committed == 1
|
|
assert recovered.errors == 0
|
|
assert not old_parent.exists()
|
|
assert not old_thumb_parent.exists()
|
|
assert service.get_job(job_id).state == "committed"
|
|
|
|
|
|
def test_preflight_rejects_active_uncommitted_job_for_same_image(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-d.png"
|
|
_save_record(records, image_name=image_name, subfolder="", created_at="2024-01-02 03:04:05.000")
|
|
service.image_files.save(Image.new("RGB", (16, 16), "yellow"), image_name=image_name)
|
|
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
service.create_move_job(moves)
|
|
|
|
with pytest.raises(ValueError, match="active image move job"):
|
|
service.plan_batch(last_image_name="", limit=100)
|
|
|
|
|
|
def test_create_move_job_rejects_second_active_job_from_stale_plan(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-active-race.png"
|
|
_save_image(service, records, image_name, "", "2024-01-03 03:04:05.000", "yellow")
|
|
|
|
stale_plan_a = service.plan_batch(last_image_name="", limit=100)
|
|
stale_plan_b = service.plan_batch(last_image_name="", limit=100)
|
|
service.create_move_job(stale_plan_a)
|
|
|
|
with pytest.raises(ValueError, match="active image move job"):
|
|
service.create_move_job(stale_plan_b)
|
|
|
|
|
|
def test_startup_recovery_completes_planned_job_before_any_file_move(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-e.png"
|
|
_save_image(service, records, image_name, "", "2024-03-04 05:06:07.000", "purple")
|
|
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
job_id = service.create_move_job(moves)
|
|
|
|
recovered_once = service.startup_recovery()
|
|
recovered_twice = service.startup_recovery()
|
|
|
|
assert recovered_once.committed == 1
|
|
assert recovered_once.errors == 0
|
|
assert recovered_twice.committed == 0
|
|
assert recovered_twice.errors == 0
|
|
assert records.get(image_name).image_subfolder == "2024/03/04"
|
|
assert service.get_job(job_id).state == "committed"
|
|
assert _job_item_states(service, job_id) == {image_name: "committed"}
|
|
|
|
|
|
def test_background_recovery_can_start_when_journal_job_is_active(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-background-recovery.png"
|
|
_save_image(service, records, image_name, "", "2024-03-05 05:06:07.000", "purple")
|
|
job_id = service.create_move_job(service.plan_batch(last_image_name="", limit=100))
|
|
|
|
status = service.start_background_recovery()
|
|
assert status.is_running is True
|
|
assert status.operation == "recovery"
|
|
|
|
assert service._future is not None
|
|
service._future.result(timeout=5)
|
|
|
|
assert records.get(image_name).image_subfolder == "2024/03/05"
|
|
assert service.get_job(job_id).state == "committed"
|
|
|
|
|
|
def test_start_runs_recovery_before_normal_operation(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-startup-recovery.png"
|
|
_save_image(service, records, image_name, "", "2024-03-05 05:06:07.000", "purple")
|
|
job_id = service.create_move_job(service.plan_batch(last_image_name="", limit=100))
|
|
service.perform_filesystem_moves(job_id)
|
|
|
|
service.start(MagicMock())
|
|
|
|
assert records.get(image_name).image_subfolder == "2024/03/05"
|
|
assert service.get_job(job_id).state == "committed"
|
|
assert service.is_maintenance_active() is False
|
|
|
|
|
|
def test_start_leaves_maintenance_active_when_recovery_remains_incomplete(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-startup-recovery-retry.png"
|
|
_save_image(service, records, image_name, "", "2024-03-05 05:06:07.000", "purple")
|
|
service.create_move_job(service.plan_batch(last_image_name="", limit=100))
|
|
|
|
with patch.object(service, "complete_partial_filesystem_moves", side_effect=OSError("temporary failure")):
|
|
service.start(MagicMock())
|
|
|
|
assert records.get(image_name).image_subfolder == ""
|
|
assert service.is_maintenance_active() is True
|
|
|
|
|
|
@pytest.mark.parametrize(("pending", "in_progress"), [(1, 0), (0, 1)])
|
|
def test_background_move_rejects_active_queue_work(tmp_path: Path, pending: int, in_progress: int) -> None:
|
|
service, _records = _service(tmp_path, strategy="date")
|
|
invoker = MagicMock()
|
|
invoker.services.session_queue.get_queue_status.return_value = SessionQueueStatus(
|
|
queue_id=DEFAULT_QUEUE_ID,
|
|
item_id=None,
|
|
batch_id=None,
|
|
session_id=None,
|
|
pending=pending,
|
|
in_progress=in_progress,
|
|
waiting=0,
|
|
completed=0,
|
|
failed=0,
|
|
canceled=0,
|
|
total=1,
|
|
)
|
|
service.start(invoker)
|
|
|
|
with pytest.raises(ImageMoveQueueActive, match="queue work is active"):
|
|
service.start_background_move_all()
|
|
|
|
|
|
def test_background_move_is_reserved_before_queue_check(tmp_path: Path) -> None:
|
|
service, _records = _service(tmp_path, strategy="date")
|
|
invoker = MagicMock()
|
|
|
|
def get_queue_status(queue_id: str) -> SessionQueueStatus:
|
|
assert queue_id == DEFAULT_QUEUE_ID
|
|
assert service.is_maintenance_active() is True
|
|
return SessionQueueStatus(
|
|
queue_id=DEFAULT_QUEUE_ID,
|
|
item_id=None,
|
|
batch_id=None,
|
|
session_id=None,
|
|
pending=1,
|
|
in_progress=0,
|
|
waiting=0,
|
|
completed=0,
|
|
failed=0,
|
|
canceled=0,
|
|
total=1,
|
|
)
|
|
|
|
invoker.services.session_queue.get_queue_status.side_effect = get_queue_status
|
|
service.start(invoker)
|
|
|
|
with pytest.raises(ImageMoveQueueActive, match="queue work is active"):
|
|
service.start_background_move_all()
|
|
|
|
assert service.is_maintenance_active() is False
|
|
|
|
|
|
def test_maintenance_is_active_while_background_job_or_uncommitted_journal_exists(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-maintenance-active.png"
|
|
_save_image(service, records, image_name, "", "2024-03-05 05:06:07.000", "purple")
|
|
service.create_move_job(service.plan_batch(last_image_name="", limit=100))
|
|
|
|
assert service.is_maintenance_active() is True
|
|
|
|
release_worker = threading.Event()
|
|
|
|
def wait_for_release() -> None:
|
|
release_worker.wait(timeout=5)
|
|
|
|
service._start_background_operation("recovery", wait_for_release)
|
|
try:
|
|
assert service.is_maintenance_active() is True
|
|
finally:
|
|
release_worker.set()
|
|
assert service._future is not None
|
|
service._future.result(timeout=5)
|
|
|
|
|
|
def test_background_worker_error_is_exposed_in_status(tmp_path: Path) -> None:
|
|
service, _records = _service(tmp_path, strategy="date")
|
|
started_worker = threading.Event()
|
|
release_worker = threading.Event()
|
|
|
|
def raise_error() -> None:
|
|
started_worker.set()
|
|
release_worker.wait(timeout=5)
|
|
raise RuntimeError("background failed")
|
|
|
|
status = service._start_background_operation("move_all", raise_error)
|
|
assert started_worker.wait(timeout=5) is True
|
|
assert status.is_running is True
|
|
|
|
assert service._future is not None
|
|
release_worker.set()
|
|
service._future.result(timeout=5)
|
|
|
|
status = service.get_background_status()
|
|
assert status.is_running is False
|
|
assert status.operation is None
|
|
assert status.last_error == "background failed"
|
|
|
|
|
|
def test_stop_waits_for_active_background_job_without_recording_error(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-background-stop.png"
|
|
_save_image(service, records, image_name, "", "2024-03-05 05:06:07.000", "purple")
|
|
job_id = service.create_move_job(service.plan_batch(last_image_name="", limit=100))
|
|
release_worker = threading.Event()
|
|
|
|
def wait_for_shutdown() -> None:
|
|
release_worker.wait(timeout=5)
|
|
|
|
service._start_background_operation("recovery", wait_for_shutdown)
|
|
|
|
stop_thread = threading.Thread(target=service.stop)
|
|
stop_thread.start()
|
|
assert stop_thread.is_alive()
|
|
|
|
release_worker.set()
|
|
stop_thread.join(timeout=5)
|
|
|
|
assert not stop_thread.is_alive()
|
|
assert service.get_job(job_id).error_message is None
|
|
assert service.get_background_status().last_error is None
|
|
|
|
|
|
def test_startup_recovery_completes_partial_multi_image_move(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
_save_image(service, records, "image-f.png", "", "2024-04-05 06:07:08.000", "orange")
|
|
_save_image(service, records, "image-g.png", "", "2024-04-06 06:07:08.000", "cyan")
|
|
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
job_id = service.create_move_job(moves)
|
|
first_move = moves[0]
|
|
first_move.new_path.parent.mkdir(parents=True, exist_ok=True)
|
|
first_move.new_thumbnail_path.parent.mkdir(parents=True, exist_ok=True)
|
|
first_move.old_path.replace(first_move.new_path)
|
|
first_move.old_thumbnail_path.replace(first_move.new_thumbnail_path)
|
|
|
|
recovered_once = service.startup_recovery()
|
|
recovered_twice = service.startup_recovery()
|
|
|
|
assert recovered_once.committed == 2
|
|
assert recovered_once.errors == 0
|
|
assert recovered_twice.committed == 0
|
|
assert recovered_twice.errors == 0
|
|
assert records.get("image-f.png").image_subfolder == "2024/04/05"
|
|
assert records.get("image-g.png").image_subfolder == "2024/04/06"
|
|
assert service.get_job(job_id).state == "committed"
|
|
assert _job_item_states(service, job_id) == {"image-f.png": "committed", "image-g.png": "committed"}
|
|
|
|
|
|
def test_startup_recovery_marks_committed_after_db_update_but_before_journal_commit(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-h.png"
|
|
_save_image(service, records, image_name, "", "2024-05-06 07:08:09.000", "pink")
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
job_id = service.create_move_job(moves)
|
|
service.perform_filesystem_moves(job_id)
|
|
|
|
with service._db.transaction() as cursor:
|
|
cursor.execute(
|
|
"UPDATE images SET image_subfolder = ? WHERE image_name = ?;",
|
|
("2024/05/06", image_name),
|
|
)
|
|
|
|
recovered_once = service.startup_recovery()
|
|
recovered_twice = service.startup_recovery()
|
|
|
|
assert recovered_once.committed == 1
|
|
assert recovered_once.errors == 0
|
|
assert recovered_twice.committed == 0
|
|
assert recovered_twice.errors == 0
|
|
assert records.get(image_name).image_subfolder == "2024/05/06"
|
|
assert service.get_job(job_id).state == "committed"
|
|
assert _job_item_states(service, job_id) == {image_name: "committed"}
|
|
|
|
|
|
def test_startup_recovery_marks_error_when_both_old_and_new_full_size_files_exist(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-i.png"
|
|
_save_image(service, records, image_name, "", "2024-07-08 09:10:11.000", "red")
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
job_id = service.create_move_job(moves)
|
|
move = moves[0]
|
|
move.new_path.parent.mkdir(parents=True, exist_ok=True)
|
|
copy2(move.old_path, move.new_path)
|
|
|
|
recovered = service.startup_recovery()
|
|
|
|
assert recovered.committed == 0
|
|
assert recovered.errors == 1
|
|
assert records.get(image_name).image_subfolder == ""
|
|
assert service.get_job(job_id).state == "error"
|
|
assert _job_item_states(service, job_id) == {image_name: "error"}
|
|
|
|
|
|
def test_startup_recovery_marks_error_when_neither_old_nor_new_full_size_file_exists(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-j.png"
|
|
_save_image(service, records, image_name, "", "2024-08-09 10:11:12.000", "blue")
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
job_id = service.create_move_job(moves)
|
|
moves[0].old_path.unlink()
|
|
|
|
recovered = service.startup_recovery()
|
|
|
|
assert recovered.committed == 0
|
|
assert recovered.errors == 1
|
|
assert records.get(image_name).image_subfolder == ""
|
|
assert service.get_job(job_id).state == "error"
|
|
assert _job_item_states(service, job_id) == {image_name: "error"}
|
|
|
|
|
|
def test_startup_recovery_keeps_job_recoverable_after_ordinary_exception(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-k.png"
|
|
_save_image(service, records, image_name, "", "2024-09-10 11:12:13.000", "white")
|
|
job_id = service.create_move_job(service.plan_batch(last_image_name="", limit=100))
|
|
|
|
with patch.object(service, "complete_partial_filesystem_moves", side_effect=OSError("temporary failure")):
|
|
recovered = service.startup_recovery()
|
|
|
|
assert recovered.committed == 0
|
|
assert recovered.errors == 1
|
|
job = service.get_job(job_id)
|
|
assert job.state == "planned"
|
|
assert job.error_message == "temporary failure"
|
|
|
|
recovered_retry = service.startup_recovery()
|
|
|
|
assert recovered_retry.committed == 1
|
|
assert recovered_retry.errors == 0
|
|
assert records.get(image_name).image_subfolder == "2024/09/10"
|
|
assert service.get_job(job_id).state == "committed"
|
|
|
|
|
|
def test_startup_recovery_regenerates_thumbnail_when_old_and_new_thumbnails_exist(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-l.png"
|
|
_save_image(service, records, image_name, "", "2024-10-11 12:13:14.000", "black")
|
|
moves = service.plan_batch(last_image_name="", limit=100)
|
|
job_id = service.create_move_job(moves)
|
|
move = moves[0]
|
|
move.new_path.parent.mkdir(parents=True, exist_ok=True)
|
|
move.new_thumbnail_path.parent.mkdir(parents=True, exist_ok=True)
|
|
move.old_path.replace(move.new_path)
|
|
copy2(move.old_thumbnail_path, move.new_thumbnail_path)
|
|
|
|
recovered = service.startup_recovery()
|
|
|
|
assert recovered.committed == 1
|
|
assert recovered.errors == 0
|
|
assert records.get(image_name).image_subfolder == "2024/10/11"
|
|
assert move.new_thumbnail_path.exists()
|
|
assert not move.old_thumbnail_path.exists()
|
|
assert service.get_job(job_id).state == "committed"
|
|
|
|
|
|
def test_preflight_rejects_duplicate_thumbnail_destination_paths(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
_save_image(service, records, "same-name.jpg", "", "2024-12-13 14:15:16.000", "red")
|
|
_save_image(service, records, "same-name.png", "", "2024-12-13 14:15:16.000", "green")
|
|
|
|
with pytest.raises(ValueError, match="Duplicate destination thumbnail path"):
|
|
service.plan_batch(last_image_name="", limit=100)
|
|
|
|
|
|
def test_successful_filesystem_move_fsyncs_files_and_directories(tmp_path: Path) -> None:
|
|
service, records = _service(tmp_path, strategy="date")
|
|
image_name = "image-m.png"
|
|
_save_image(service, records, image_name, "", "2025-01-02 03:04:05.000", "blue")
|
|
job_id = service.create_move_job(service.plan_batch(last_image_name="", limit=100))
|
|
|
|
with (
|
|
patch.object(service, "_fsync_file") as fsync_file,
|
|
patch.object(service, "_fsync_dir") as fsync_dir,
|
|
):
|
|
service.perform_filesystem_moves(job_id)
|
|
|
|
moved = service._get_items(job_id)[0]
|
|
fsync_file.assert_any_call(moved.new_path)
|
|
fsync_file.assert_any_call(moved.new_thumbnail_path)
|
|
fsync_dir.assert_any_call(moved.new_path.parent)
|
|
fsync_dir.assert_any_call(moved.old_path.parent)
|
|
fsync_dir.assert_any_call(moved.new_thumbnail_path.parent)
|
|
fsync_dir.assert_any_call(moved.old_thumbnail_path.parent)
|
|
|
|
|
|
def test_fsync_dir_ignores_platform_close_failures(tmp_path: Path) -> None:
|
|
service, _records = _service(tmp_path, strategy="date")
|
|
|
|
with (
|
|
patch("invokeai.app.services.image_moves.image_moves_default.os.open", return_value=123),
|
|
patch(
|
|
"invokeai.app.services.image_moves.image_moves_default.os.fsync",
|
|
side_effect=OSError(9, "Bad file descriptor"),
|
|
),
|
|
patch(
|
|
"invokeai.app.services.image_moves.image_moves_default.os.close",
|
|
side_effect=OSError(9, "Bad file descriptor"),
|
|
),
|
|
):
|
|
service._fsync_dir(tmp_path)
|
|
|
|
|
|
def test_fsync_file_ignores_platform_fsync_failures(tmp_path: Path) -> None:
|
|
service, _records = _service(tmp_path, strategy="date")
|
|
path = tmp_path / "image.png"
|
|
path.write_bytes(b"test")
|
|
|
|
with patch(
|
|
"invokeai.app.services.image_moves.image_moves_default.os.fsync",
|
|
side_effect=OSError(9, "Bad file descriptor"),
|
|
):
|
|
service._fsync_file(path)
|