"""Tests for S3 storage implementation.""" import io from unittest.mock import MagicMock, patch import pytest from application.storage.s3 import S3Storage from botocore.exceptions import ClientError @pytest.fixture def mock_boto3_client(): """Mock boto3.client to isolate S3 client creation.""" with patch("boto3.client") as mock_client: s3_mock = MagicMock() mock_client.return_value = s3_mock yield s3_mock @pytest.fixture def s3_storage(mock_boto3_client): """Create S3Storage instance with mocked boto3 client.""" return S3Storage(bucket_name="test-bucket") class TestS3StorageInitialization: """Test S3Storage initialization and configuration.""" @pytest.mark.unit def test_init_with_default_bucket(self): """Should use default bucket name when none provided.""" with patch("boto3.client"): storage = S3Storage() assert storage.bucket_name == "docsgpt-test-bucket" @pytest.mark.unit def test_init_with_custom_bucket(self): """Should use provided bucket name.""" with patch("boto3.client"): storage = S3Storage(bucket_name="custom-bucket") assert storage.bucket_name == "custom-bucket" @pytest.mark.unit def test_init_creates_boto3_client_with_s3_credentials(self): """Should create boto3 S3 client with dedicated S3_* credentials.""" with patch("boto3.client") as mock_client, patch( "application.storage.s3.settings" ) as mock_settings: mock_settings.S3_BUCKET_NAME = "docsgpt-test-bucket" mock_settings.S3_ACCESS_KEY_ID = "test-key" mock_settings.S3_SECRET_ACCESS_KEY = "test-secret" mock_settings.S3_REGION = "us-west-2" mock_settings.S3_ENDPOINT_URL = None mock_settings.S3_PATH_STYLE = False mock_settings.SAGEMAKER_ACCESS_KEY = None mock_settings.SAGEMAKER_SECRET_KEY = None mock_settings.SAGEMAKER_REGION = None S3Storage() mock_client.assert_called_once_with( "s3", aws_access_key_id="test-key", aws_secret_access_key="test-secret", region_name="us-west-2", ) @pytest.mark.unit def test_init_falls_back_to_sagemaker_credentials(self): """Should fall back to deprecated SAGEMAKER_* credentials when S3_* unset.""" with patch("boto3.client") as mock_client, patch( "application.storage.s3.settings" ) as mock_settings, patch("application.storage.s3.logger") as mock_logger: mock_settings.S3_BUCKET_NAME = "docsgpt-test-bucket" mock_settings.S3_ACCESS_KEY_ID = None mock_settings.S3_SECRET_ACCESS_KEY = None mock_settings.S3_REGION = None mock_settings.S3_ENDPOINT_URL = None mock_settings.S3_PATH_STYLE = False mock_settings.SAGEMAKER_ACCESS_KEY = "legacy-key" mock_settings.SAGEMAKER_SECRET_KEY = "legacy-secret" mock_settings.SAGEMAKER_REGION = "eu-central-1" S3Storage() mock_client.assert_called_once_with( "s3", aws_access_key_id="legacy-key", aws_secret_access_key="legacy-secret", region_name="eu-central-1", ) mock_logger.warning.assert_called_once() @pytest.mark.unit def test_init_with_custom_endpoint_and_path_style(self): """Should pass endpoint_url and path-style config for S3-compatible services.""" with patch("boto3.client") as mock_client, patch( "application.storage.s3.settings" ) as mock_settings: mock_settings.S3_BUCKET_NAME = "my-bucket" mock_settings.S3_ACCESS_KEY_ID = "key" mock_settings.S3_SECRET_ACCESS_KEY = "secret" mock_settings.S3_REGION = "auto" mock_settings.S3_ENDPOINT_URL = "https://account.r2.cloudflarestorage.com" mock_settings.S3_PATH_STYLE = True mock_settings.SAGEMAKER_ACCESS_KEY = None mock_settings.SAGEMAKER_SECRET_KEY = None mock_settings.SAGEMAKER_REGION = None S3Storage() _, kwargs = mock_client.call_args assert kwargs["endpoint_url"] == "https://account.r2.cloudflarestorage.com" assert kwargs["config"].s3 == {"addressing_style": "path"} class TestS3StorageSaveFile: """Test file saving functionality.""" @pytest.mark.unit def test_save_file_uploads_to_s3(self, s3_storage, mock_boto3_client): """Should upload file to S3 with correct parameters.""" file_data = io.BytesIO(b"test content") path = "documents/test.txt" s3_storage.region = "us-east-1" result = s3_storage.save_file(file_data, path) mock_boto3_client.upload_fileobj.assert_called_once_with( file_data, "test-bucket", path, ExtraArgs={"StorageClass": "INTELLIGENT_TIERING"}, ) assert result == { "storage_type": "s3", "bucket_name": "test-bucket", "uri": "s3://test-bucket/documents/test.txt", "region": "us-east-1", } @pytest.mark.unit def test_save_file_with_custom_storage_class(self, s3_storage, mock_boto3_client): """Should use custom storage class when provided.""" file_data = io.BytesIO(b"test content") path = "documents/test.txt" with patch("application.storage.s3.settings") as mock_settings: mock_settings.SAGEMAKER_REGION = "us-east-1" s3_storage.save_file(file_data, path, storage_class="STANDARD") mock_boto3_client.upload_fileobj.assert_called_once_with( file_data, "test-bucket", path, ExtraArgs={"StorageClass": "STANDARD"} ) @pytest.mark.unit def test_save_file_propagates_client_error(self, s3_storage, mock_boto3_client): """Should propagate ClientError when upload fails.""" file_data = io.BytesIO(b"test content") path = "documents/test.txt" mock_boto3_client.upload_fileobj.side_effect = ClientError( {"Error": {"Code": "AccessDenied", "Message": "Access denied"}}, "upload_fileobj", ) with pytest.raises(ClientError): s3_storage.save_file(file_data, path) class TestS3StorageFileExists: """Test file existence checking.""" @pytest.mark.unit def test_file_exists_returns_true_when_file_found( self, s3_storage, mock_boto3_client ): """Should return True when head_object succeeds.""" path = "documents/test.txt" mock_boto3_client.head_object.return_value = {"ContentLength": 100} result = s3_storage.file_exists(path) assert result is True mock_boto3_client.head_object.assert_called_once_with( Bucket="test-bucket", Key=path ) @pytest.mark.unit def test_file_exists_returns_false_on_client_error( self, s3_storage, mock_boto3_client ): """Should return False when head_object raises ClientError.""" path = "documents/nonexistent.txt" mock_boto3_client.head_object.side_effect = ClientError( {"Error": {"Code": "NoSuchKey", "Message": "Not found"}}, "head_object" ) result = s3_storage.file_exists(path) assert result is False class TestS3StorageGetFile: """Test file retrieval functionality.""" @pytest.mark.unit def test_get_file_downloads_and_returns_file_object( self, s3_storage, mock_boto3_client ): """Should download file from S3 and return BytesIO object.""" path = "documents/test.txt" test_content = b"file content" mock_boto3_client.head_object.return_value = {} def mock_download(bucket, key, file_obj): file_obj.write(test_content) mock_boto3_client.download_fileobj.side_effect = mock_download result = s3_storage.get_file(path) assert isinstance(result, io.BytesIO) assert result.read() == test_content mock_boto3_client.download_fileobj.assert_called_once() @pytest.mark.unit def test_get_file_raises_error_when_file_not_found( self, s3_storage, mock_boto3_client ): """Should raise FileNotFoundError when file doesn't exist.""" path = "documents/nonexistent.txt" mock_boto3_client.head_object.side_effect = ClientError( {"Error": {"Code": "NoSuchKey", "Message": "Not found"}}, "head_object" ) with pytest.raises(FileNotFoundError, match="File not found"): s3_storage.get_file(path) class TestS3StorageDeleteFile: """Test file deletion functionality.""" @pytest.mark.unit def test_delete_file_returns_true_on_success(self, s3_storage, mock_boto3_client): """Should return True when deletion succeeds.""" path = "documents/test.txt" mock_boto3_client.delete_object.return_value = {} result = s3_storage.delete_file(path) assert result is True mock_boto3_client.delete_object.assert_called_once_with( Bucket="test-bucket", Key=path ) @pytest.mark.unit def test_delete_file_returns_false_on_client_error( self, s3_storage, mock_boto3_client ): """Should return False when deletion fails with ClientError.""" path = "documents/test.txt" mock_boto3_client.delete_object.side_effect = ClientError( {"Error": {"Code": "AccessDenied", "Message": "Access denied"}}, "delete_object", ) result = s3_storage.delete_file(path) assert result is False class TestS3StorageListFiles: """Test directory listing functionality.""" @pytest.mark.unit def test_list_files_returns_all_keys_with_prefix( self, s3_storage, mock_boto3_client ): """Should return all file keys matching the directory prefix.""" directory = "documents/" paginator_mock = MagicMock() mock_boto3_client.get_paginator.return_value = paginator_mock paginator_mock.paginate.return_value = [ { "Contents": [ {"Key": "documents/file1.txt"}, {"Key": "documents/file2.txt"}, {"Key": "documents/subdir/file3.txt"}, ] } ] result = s3_storage.list_files(directory) assert len(result) == 3 assert "documents/file1.txt" in result assert "documents/file2.txt" in result assert "documents/subdir/file3.txt" in result mock_boto3_client.get_paginator.assert_called_once_with("list_objects_v2") paginator_mock.paginate.assert_called_once_with( Bucket="test-bucket", Prefix="documents/" ) @pytest.mark.unit def test_list_files_returns_empty_list_when_no_contents( self, s3_storage, mock_boto3_client ): """Should return empty list when directory has no files.""" directory = "empty/" paginator_mock = MagicMock() mock_boto3_client.get_paginator.return_value = paginator_mock paginator_mock.paginate.return_value = [{}] result = s3_storage.list_files(directory) assert result == [] class TestS3StorageProcessFile: """Test file processing functionality.""" @pytest.mark.unit def test_process_file_downloads_and_processes_file( self, s3_storage, mock_boto3_client ): """Should download file to temp location and call processor function.""" path = "documents/test.txt" mock_boto3_client.head_object.return_value = {} with patch("tempfile.NamedTemporaryFile") as mock_temp: mock_file = MagicMock() mock_file.name = "/tmp/test_file" mock_temp.return_value.__enter__.return_value = mock_file processor_func = MagicMock(return_value="processed") result = s3_storage.process_file(path, processor_func, extra_arg="value") assert result == "processed" processor_func.assert_called_once_with( local_path="/tmp/test_file", extra_arg="value" ) mock_boto3_client.download_fileobj.assert_called_once() @pytest.mark.unit def test_process_file_raises_error_when_file_not_found( self, s3_storage, mock_boto3_client ): """Should raise FileNotFoundError when file doesn't exist.""" path = "documents/nonexistent.txt" mock_boto3_client.head_object.side_effect = ClientError( {"Error": {"Code": "NoSuchKey", "Message": "Not found"}}, "head_object" ) processor_func = MagicMock() with pytest.raises(FileNotFoundError, match="File not found in S3"): s3_storage.process_file(path, processor_func) class TestS3StorageIsDirectory: """Test directory checking functionality.""" @pytest.mark.unit def test_is_directory_returns_true_when_objects_exist( self, s3_storage, mock_boto3_client ): """Should return True when objects exist with the directory prefix.""" path = "documents/" mock_boto3_client.list_objects_v2.return_value = { "Contents": [{"Key": "documents/file1.txt"}] } result = s3_storage.is_directory(path) assert result is True mock_boto3_client.list_objects_v2.assert_called_once_with( Bucket="test-bucket", Prefix="documents/", MaxKeys=1 ) @pytest.mark.unit def test_is_directory_returns_false_when_no_objects_exist( self, s3_storage, mock_boto3_client ): """Should return False when no objects exist with the directory prefix.""" path = "nonexistent/" mock_boto3_client.list_objects_v2.return_value = {} result = s3_storage.is_directory(path) assert result is False class TestS3StorageRemoveDirectory: """Test directory removal functionality.""" @pytest.mark.unit def test_remove_directory_deletes_all_objects(self, s3_storage, mock_boto3_client): """Should delete all objects with the directory prefix.""" directory = "documents/" paginator_mock = MagicMock() mock_boto3_client.get_paginator.return_value = paginator_mock paginator_mock.paginate.return_value = [ { "Contents": [ {"Key": "documents/file1.txt"}, {"Key": "documents/file2.txt"}, ] } ] mock_boto3_client.delete_objects.return_value = { "Deleted": [{"Key": "documents/file1.txt"}, {"Key": "documents/file2.txt"}] } result = s3_storage.remove_directory(directory) assert result is True mock_boto3_client.delete_objects.assert_called_once() call_args = mock_boto3_client.delete_objects.call_args[1] assert call_args["Bucket"] == "test-bucket" assert len(call_args["Delete"]["Objects"]) == 2 @pytest.mark.unit def test_remove_directory_returns_false_when_empty( self, s3_storage, mock_boto3_client ): """Should return False when directory is empty (no objects to delete).""" directory = "empty/" paginator_mock = MagicMock() mock_boto3_client.get_paginator.return_value = paginator_mock paginator_mock.paginate.return_value = [{}] result = s3_storage.remove_directory(directory) assert result is False mock_boto3_client.delete_objects.assert_not_called() @pytest.mark.unit def test_remove_directory_returns_false_on_client_error( self, s3_storage, mock_boto3_client ): """Should return False when deletion fails with ClientError.""" directory = "documents/" paginator_mock = MagicMock() mock_boto3_client.get_paginator.return_value = paginator_mock paginator_mock.paginate.return_value = [ {"Contents": [{"Key": "documents/file1.txt"}]} ] mock_boto3_client.delete_objects.side_effect = ClientError( {"Error": {"Code": "AccessDenied", "Message": "Access denied"}}, "delete_objects", ) result = s3_storage.remove_directory(directory) assert result is False @pytest.mark.unit def test_remove_directory_returns_false_on_delete_errors( self, s3_storage, mock_boto3_client ): """Should return False when delete_objects response contains Errors.""" directory = "documents/" paginator_mock = MagicMock() mock_boto3_client.get_paginator.return_value = paginator_mock paginator_mock.paginate.return_value = [ {"Contents": [{"Key": "documents/file1.txt"}]} ] mock_boto3_client.delete_objects.return_value = { "Errors": [{"Key": "documents/file1.txt", "Code": "InternalError"}] } result = s3_storage.remove_directory(directory) assert result is False class TestS3StorageDirectorySlashHandling: """Test that directories without trailing slashes get them added.""" @pytest.mark.unit def test_list_files_adds_trailing_slash(self, s3_storage, mock_boto3_client): """Should add trailing slash when listing directory without one.""" paginator_mock = MagicMock() mock_boto3_client.get_paginator.return_value = paginator_mock paginator_mock.paginate.return_value = [{}] s3_storage.list_files("documents") paginator_mock.paginate.assert_called_once_with( Bucket="test-bucket", Prefix="documents/" ) @pytest.mark.unit def test_list_files_empty_directory_string(self, s3_storage, mock_boto3_client): """Empty string directory should not get a slash added.""" paginator_mock = MagicMock() mock_boto3_client.get_paginator.return_value = paginator_mock paginator_mock.paginate.return_value = [{}] s3_storage.list_files("") paginator_mock.paginate.assert_called_once_with( Bucket="test-bucket", Prefix="" ) @pytest.mark.unit def test_is_directory_adds_trailing_slash(self, s3_storage, mock_boto3_client): """Should add trailing slash for is_directory check.""" mock_boto3_client.list_objects_v2.return_value = {} s3_storage.is_directory("docs") mock_boto3_client.list_objects_v2.assert_called_once_with( Bucket="test-bucket", Prefix="docs/", MaxKeys=1 ) @pytest.mark.unit def test_remove_directory_adds_trailing_slash(self, s3_storage, mock_boto3_client): """Should add trailing slash for remove_directory.""" paginator_mock = MagicMock() mock_boto3_client.get_paginator.return_value = paginator_mock paginator_mock.paginate.return_value = [{}] s3_storage.remove_directory("docs") paginator_mock.paginate.assert_called_once() call_kwargs = paginator_mock.paginate.call_args[1] assert call_kwargs["Prefix"] == "docs/" class TestS3StorageProcessFileError: """Test error handling in process_file.""" @pytest.mark.unit def test_process_file_propagates_processor_error( self, s3_storage, mock_boto3_client ): """Should propagate errors from the processor function.""" path = "documents/test.txt" mock_boto3_client.head_object.return_value = {} with patch("tempfile.NamedTemporaryFile") as mock_temp: mock_file = MagicMock() mock_file.name = "/tmp/test_file" mock_temp.return_value.__enter__.return_value = mock_file processor_func = MagicMock(side_effect=RuntimeError("Process failed")) with pytest.raises(RuntimeError, match="Process failed"): s3_storage.process_file(path, processor_func)