wehub-resource-sync
655 lines
23 KiB
Python
655 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
"""End-to-end tests for the Docker sandbox backend.
|
|
|
|
These tests require a running Docker daemon. They exercise the full container
|
|
lifecycle: image build, container start, command execution, file isolation,
|
|
network isolation, resource limits, and cleanup.
|
|
|
|
Run directly:
|
|
python3 scripts/test_docker_sandbox_e2e.py
|
|
|
|
Or via pytest (skipped automatically when Docker is unavailable):
|
|
uv run pytest scripts/test_docker_sandbox_e2e.py -v
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bootstrap: ensure the src package is importable when run as a script
|
|
# ---------------------------------------------------------------------------
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
|
|
from openharness.config.settings import (
|
|
DockerSandboxSettings,
|
|
SandboxNetworkSettings,
|
|
SandboxSettings,
|
|
Settings,
|
|
)
|
|
from openharness.sandbox.docker_backend import DockerSandboxSession, get_docker_availability
|
|
from openharness.sandbox.docker_image import ensure_image_available
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Skip condition: Docker daemon must be reachable
|
|
# ---------------------------------------------------------------------------
|
|
_DOCKER = shutil.which("docker")
|
|
_DOCKER_OK = False
|
|
if _DOCKER:
|
|
try:
|
|
subprocess.run([_DOCKER, "info"], capture_output=True, timeout=10, check=True)
|
|
_DOCKER_OK = True
|
|
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError):
|
|
pass
|
|
|
|
_SKIP_REASON = "Docker daemon is not available"
|
|
_E2E_IMAGE = "openharness-sandbox-e2e:latest"
|
|
|
|
pytestmark = pytest.mark.skipif(not _DOCKER_OK, reason=_SKIP_REASON)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _settings(*, network_domains: list[str] | None = None, **docker_kw) -> Settings:
|
|
"""Build a Settings object with Docker sandbox enabled."""
|
|
return Settings(
|
|
sandbox=SandboxSettings(
|
|
enabled=True,
|
|
backend="docker",
|
|
fail_if_unavailable=True,
|
|
network=SandboxNetworkSettings(
|
|
allowed_domains=network_domains or [],
|
|
),
|
|
docker=DockerSandboxSettings(
|
|
image=_E2E_IMAGE,
|
|
auto_build_image=True,
|
|
**docker_kw,
|
|
),
|
|
)
|
|
)
|
|
|
|
|
|
async def _run_and_capture(session: DockerSandboxSession, argv: list[str], cwd: Path) -> tuple[int, str, str]:
|
|
"""Run a command inside the sandbox and return (returncode, stdout, stderr)."""
|
|
proc = await session.exec_command(
|
|
argv,
|
|
cwd=cwd,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
stdout, stderr = await proc.communicate()
|
|
return (
|
|
proc.returncode or 0,
|
|
stdout.decode("utf-8", errors="replace").strip(),
|
|
stderr.decode("utf-8", errors="replace").strip(),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.fixture(scope="module")
|
|
def e2e_image():
|
|
"""Ensure the E2E sandbox image exists (built once per module)."""
|
|
ok = asyncio.run(ensure_image_available(_E2E_IMAGE, auto_build=True))
|
|
if not ok:
|
|
pytest.skip(f"Could not build Docker image {_E2E_IMAGE}")
|
|
return _E2E_IMAGE
|
|
|
|
|
|
@pytest.fixture()
|
|
def project_dir():
|
|
"""Create a temporary project directory for one test."""
|
|
with tempfile.TemporaryDirectory(prefix="oh-docker-e2e-") as tmpdir:
|
|
yield Path(tmpdir)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. Image management
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestImageManagement:
|
|
def test_image_build(self, e2e_image):
|
|
"""Image should be available after the module fixture runs."""
|
|
result = subprocess.run(
|
|
[_DOCKER, "image", "inspect", e2e_image],
|
|
capture_output=True,
|
|
)
|
|
assert result.returncode == 0, "E2E image should exist after build"
|
|
|
|
def test_image_has_expected_tools(self, e2e_image):
|
|
"""Image should contain bash, rg (ripgrep), and git."""
|
|
for tool in ("bash", "rg", "git"):
|
|
result = subprocess.run(
|
|
[_DOCKER, "run", "--rm", e2e_image, "which", tool],
|
|
capture_output=True,
|
|
)
|
|
assert result.returncode == 0, f"{tool} should be available in the image"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. Container lifecycle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestContainerLifecycle:
|
|
def test_start_and_stop(self, e2e_image, project_dir):
|
|
"""Container should start, appear in docker ps, then stop cleanly."""
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-lifecycle", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
assert session.is_running
|
|
|
|
# Verify container is visible
|
|
result = subprocess.run(
|
|
[_DOCKER, "ps", "--filter", f"name={session.container_name}", "-q"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.stdout.strip(), "Container should appear in docker ps"
|
|
|
|
await session.stop()
|
|
assert not session.is_running
|
|
|
|
# Verify container is gone (--rm flag auto-removes)
|
|
result = subprocess.run(
|
|
[_DOCKER, "ps", "-a", "--filter", f"name={session.container_name}", "-q"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert not result.stdout.strip(), "Container should be removed after stop"
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_stop_sync(self, e2e_image, project_dir):
|
|
"""Synchronous stop (atexit handler) should also clean up."""
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-stopsync", cwd=project_dir,
|
|
)
|
|
|
|
async def _start():
|
|
await session.start()
|
|
|
|
asyncio.run(_start())
|
|
assert session.is_running
|
|
|
|
session.stop_sync()
|
|
assert not session.is_running
|
|
|
|
def test_availability_check(self):
|
|
"""get_docker_availability should return available=True when Docker is running."""
|
|
settings = _settings()
|
|
avail = get_docker_availability(settings)
|
|
assert avail.enabled is True
|
|
assert avail.available is True
|
|
assert avail.command is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. Command execution
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestCommandExecution:
|
|
def test_echo(self, e2e_image, project_dir):
|
|
"""Basic command should execute and return output."""
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-echo", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
rc, stdout, stderr = await _run_and_capture(
|
|
session, ["echo", "hello-from-sandbox"], project_dir,
|
|
)
|
|
assert rc == 0
|
|
assert "hello-from-sandbox" in stdout
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_exit_code_preserved(self, e2e_image, project_dir):
|
|
"""Non-zero exit codes should propagate correctly."""
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-exitcode", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
rc, _, _ = await _run_and_capture(
|
|
session, ["bash", "-c", "exit 42"], project_dir,
|
|
)
|
|
assert rc == 42
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_env_vars_passed(self, e2e_image, project_dir):
|
|
"""Environment variables should be forwarded into the container."""
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-env", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
rc, stdout, _ = await _run_and_capture(
|
|
session,
|
|
["bash", "-c", "echo $MY_TEST_VAR"],
|
|
project_dir,
|
|
)
|
|
# env passed through exec_command
|
|
proc = await session.exec_command(
|
|
["bash", "-c", "echo $MY_TEST_VAR"],
|
|
cwd=project_dir,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
env={"MY_TEST_VAR": "sandbox-value"},
|
|
)
|
|
out, _ = await proc.communicate()
|
|
assert "sandbox-value" in out.decode()
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_working_directory(self, e2e_image, project_dir):
|
|
"""Commands should run in the specified working directory."""
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-cwd", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
rc, stdout, _ = await _run_and_capture(
|
|
session, ["pwd"], project_dir,
|
|
)
|
|
assert rc == 0
|
|
assert str(project_dir.resolve()) in stdout
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. Filesystem isolation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestFilesystemIsolation:
|
|
def test_bind_mount_readable(self, e2e_image, project_dir):
|
|
"""Files in the project directory should be readable from inside the container."""
|
|
marker = project_dir / "test_marker.txt"
|
|
marker.write_text("E2E_MARKER_OK", encoding="utf-8")
|
|
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-fsread", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
rc, stdout, _ = await _run_and_capture(
|
|
session, ["cat", str(marker)], project_dir,
|
|
)
|
|
assert rc == 0
|
|
assert "E2E_MARKER_OK" in stdout
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_bind_mount_writable(self, e2e_image, project_dir):
|
|
"""Files written inside the container should appear on the host."""
|
|
output_file = project_dir / "from_container.txt"
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-fswrite", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
rc, _, _ = await _run_and_capture(
|
|
session,
|
|
["bash", "-c", f"echo CONTAINER_WROTE_THIS > {output_file}"],
|
|
project_dir,
|
|
)
|
|
assert rc == 0
|
|
assert output_file.exists(), "File written in container should exist on host"
|
|
assert "CONTAINER_WROTE_THIS" in output_file.read_text()
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_host_root_not_accessible(self, e2e_image, project_dir):
|
|
"""The container should NOT be able to read host files outside the bind mount."""
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-fsfence", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
# /etc/hostname exists on the host but should not be the same inside
|
|
# the container (container has its own /etc/hostname).
|
|
# More importantly, a path like /root/.bashrc should not be the host's.
|
|
rc, stdout, _ = await _run_and_capture(
|
|
session, ["ls", "/home"], project_dir,
|
|
)
|
|
# The container should have its own /home (with ohuser),
|
|
# not the host's /home
|
|
assert rc == 0
|
|
assert "ohuser" in stdout, (
|
|
"Container /home should contain the sandbox user, not host users"
|
|
)
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_ripgrep_inside_container(self, e2e_image, project_dir):
|
|
"""rg should work inside the container for glob/grep tool integration."""
|
|
(project_dir / "hello.py").write_text("print('hello world')\n", encoding="utf-8")
|
|
|
|
settings = _settings()
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-rg", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
rc, stdout, _ = await _run_and_capture(
|
|
session, ["rg", "--no-heading", "hello", "."], project_dir,
|
|
)
|
|
assert rc == 0
|
|
assert "hello world" in stdout
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. Network isolation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestNetworkIsolation:
|
|
def test_network_none_blocks_connectivity(self, e2e_image, project_dir):
|
|
"""With --network=none, outbound connections should fail."""
|
|
settings = _settings() # no allowed_domains -> --network=none
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-netblk", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
# Try to reach an external host; should fail
|
|
rc, _, _ = await _run_and_capture(
|
|
session,
|
|
["bash", "-c", "timeout 3 bash -c 'echo > /dev/tcp/8.8.8.8/53' 2>&1 || exit 1"],
|
|
project_dir,
|
|
)
|
|
assert rc != 0, "Network should be blocked with --network=none"
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_network_bridge_allows_connectivity(self, e2e_image, project_dir):
|
|
"""With allowed_domains set, --network=bridge is used and DNS resolves."""
|
|
settings = _settings(network_domains=["github.com"])
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-netok", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
# With bridge network, DNS resolution should work
|
|
rc, stdout, _ = await _run_and_capture(
|
|
session,
|
|
["bash", "-c", "getent hosts github.com 2>/dev/null && echo DNS_OK || echo DNS_FAIL"],
|
|
project_dir,
|
|
)
|
|
# getent may not be installed in slim image; check if we at least
|
|
# have network interfaces beyond loopback
|
|
rc2, stdout2, _ = await _run_and_capture(
|
|
session,
|
|
["bash", "-c", "cat /proc/net/route | wc -l"],
|
|
project_dir,
|
|
)
|
|
route_lines = int(stdout2.strip()) if stdout2.strip().isdigit() else 0
|
|
assert route_lines > 1, (
|
|
"Bridge network should have routing entries beyond just the header"
|
|
)
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 6. Resource limits
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestResourceLimits:
|
|
def test_cpu_limit_applied(self, e2e_image, project_dir):
|
|
"""Container should reflect the configured CPU limit."""
|
|
settings = _settings(cpu_limit=1.5)
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-cpu", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
result = subprocess.run(
|
|
[_DOCKER, "inspect", "--format", "{{.HostConfig.NanoCpus}}",
|
|
session.container_name],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
# Docker stores NanoCpus as int nanoseconds: 1.5 CPUs = 1_500_000_000
|
|
nano = int(result.stdout.strip())
|
|
assert nano == 1_500_000_000, f"Expected 1.5 CPUs (1500000000 nano), got {nano}"
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_memory_limit_applied(self, e2e_image, project_dir):
|
|
"""Container should reflect the configured memory limit."""
|
|
settings = _settings(memory_limit="256m")
|
|
session = DockerSandboxSession(
|
|
settings=settings, session_id="e2e-mem", cwd=project_dir,
|
|
)
|
|
|
|
async def _run():
|
|
await session.start()
|
|
try:
|
|
result = subprocess.run(
|
|
[_DOCKER, "inspect", "--format", "{{.HostConfig.Memory}}",
|
|
session.container_name],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
mem_bytes = int(result.stdout.strip())
|
|
expected = 256 * 1024 * 1024 # 256 MiB
|
|
assert mem_bytes == expected, f"Expected {expected} bytes, got {mem_bytes}"
|
|
finally:
|
|
await session.stop()
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 7. Session integration (start_docker_sandbox / stop_docker_sandbox)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestSessionIntegration:
|
|
def test_session_lifecycle(self, e2e_image, project_dir):
|
|
"""start_docker_sandbox / stop_docker_sandbox should manage the global session."""
|
|
from openharness.sandbox.session import (
|
|
get_docker_sandbox,
|
|
is_docker_sandbox_active,
|
|
start_docker_sandbox,
|
|
stop_docker_sandbox,
|
|
)
|
|
|
|
settings = _settings()
|
|
|
|
async def _run():
|
|
assert not is_docker_sandbox_active()
|
|
|
|
await start_docker_sandbox(settings, "e2e-session", project_dir)
|
|
assert is_docker_sandbox_active()
|
|
|
|
session = get_docker_sandbox()
|
|
assert session is not None
|
|
assert session.is_running
|
|
|
|
# Run a command through the session
|
|
rc, stdout, _ = await _run_and_capture(session, ["echo", "session-ok"], project_dir)
|
|
assert rc == 0
|
|
assert "session-ok" in stdout
|
|
|
|
await stop_docker_sandbox()
|
|
assert not is_docker_sandbox_active()
|
|
assert get_docker_sandbox() is None
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 8. shell.py integration (create_shell_subprocess routes through Docker)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestShellIntegration:
|
|
def test_create_shell_subprocess_routes_through_docker(self, e2e_image, project_dir):
|
|
"""When Docker sandbox is active, create_shell_subprocess should exec inside container."""
|
|
from openharness.sandbox.session import (
|
|
start_docker_sandbox,
|
|
stop_docker_sandbox,
|
|
)
|
|
from openharness.utils.shell import create_shell_subprocess
|
|
|
|
settings = _settings()
|
|
|
|
async def _run():
|
|
await start_docker_sandbox(settings, "e2e-shell", project_dir)
|
|
try:
|
|
process = await create_shell_subprocess(
|
|
"echo DOCKER_SHELL_OK",
|
|
cwd=project_dir,
|
|
settings=settings,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
stdout, _ = await process.communicate()
|
|
assert "DOCKER_SHELL_OK" in stdout.decode()
|
|
finally:
|
|
await stop_docker_sandbox()
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI entry point for running outside pytest
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _main() -> int:
|
|
"""Run all tests and report results."""
|
|
if not _DOCKER_OK:
|
|
print(f"SKIP: {_SKIP_REASON}")
|
|
return 0
|
|
|
|
# Collect all test classes
|
|
test_classes = [
|
|
TestImageManagement,
|
|
TestContainerLifecycle,
|
|
TestCommandExecution,
|
|
TestFilesystemIsolation,
|
|
TestNetworkIsolation,
|
|
TestResourceLimits,
|
|
TestSessionIntegration,
|
|
TestShellIntegration,
|
|
]
|
|
|
|
# Ensure image is built first
|
|
print(f"Building sandbox image {_E2E_IMAGE}...")
|
|
ok = asyncio.run(ensure_image_available(_E2E_IMAGE, auto_build=True))
|
|
if not ok:
|
|
print(f"FAIL: Could not build {_E2E_IMAGE}")
|
|
return 1
|
|
print("Image ready.\n")
|
|
|
|
passed = 0
|
|
failed = 0
|
|
errors: list[str] = []
|
|
|
|
for cls in test_classes:
|
|
instance = cls()
|
|
for attr in sorted(dir(instance)):
|
|
if not attr.startswith("test_"):
|
|
continue
|
|
method = getattr(instance, attr)
|
|
name = f"{cls.__name__}.{attr}"
|
|
try:
|
|
with tempfile.TemporaryDirectory(prefix="oh-docker-e2e-") as tmpdir:
|
|
# Inject fixtures based on parameter names
|
|
import inspect
|
|
|
|
sig = inspect.signature(method)
|
|
kwargs = {}
|
|
if "e2e_image" in sig.parameters:
|
|
kwargs["e2e_image"] = _E2E_IMAGE
|
|
if "project_dir" in sig.parameters:
|
|
kwargs["project_dir"] = Path(tmpdir)
|
|
method(**kwargs)
|
|
print(f" PASS {name}")
|
|
passed += 1
|
|
except Exception as exc:
|
|
print(f" FAIL {name}: {exc}")
|
|
errors.append(f"{name}: {exc}")
|
|
failed += 1
|
|
|
|
print(f"\n{'=' * 60}")
|
|
print(f"Results: {passed} passed, {failed} failed")
|
|
if errors:
|
|
print("\nFailures:")
|
|
for e in errors:
|
|
print(f" - {e}")
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(_main())
|