chore: import upstream snapshot with attribution
Integ / changes (push) Has been skipped
Pre-commit / pre-commit (push) Failing after 1s
CLI exit codes / changes (push) Has been skipped
Test (Install) / changes (push) Has been skipped
Test (Python) / changes (push) Has been skipped
Test (TypeScript) / changes (push) Has been skipped
CLI exit codes / cli-gate (push) Has been cancelled
Test (Install) / test-install-gate (push) Has been cancelled
Integ / integ-gate (push) Has been cancelled
Test (Python) / test-python-gate (push) Has been cancelled
Test (TypeScript) / test-typescript-gate (push) Has been cancelled
Test (Install) / python-minimal (3.12) (push) Has been cancelled
Test (Install) / python-minimal (3.11) (push) Has been cancelled
Test (Install) / python-extra (agno, mirage.agents.agno) (push) Has been cancelled
Test (Install) / python-extra (chroma, mirage.resource.chroma) (push) Has been cancelled
Test (Install) / python-extra (pdf, mirage.core.filetype.pdf) (push) Has been cancelled
Integ / integ (push) Has been cancelled
Integ / integ-database (push) Has been cancelled
Integ / integ-database-ts (push) Has been cancelled
Integ / integ-data (push) Has been cancelled
Integ / integ-ssh (push) Has been cancelled
Integ / integ-ssh-ts (push) Has been cancelled
Test (Python) / audit (push) Has been cancelled
Test (TypeScript) / test (push) Has been cancelled
Test (TypeScript) / python-fs-shim (push) Has been cancelled
CLI exit codes / Python CLI (push) Has been cancelled
CLI exit codes / TypeScript CLI (push) Has been cancelled
CLI exit codes / Cross-language snapshot interop (push) Has been cancelled
Test (Python) / test (push) Has been cancelled
Test (Python) / import-isolation (deepagents, openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Python) / import-isolation (deepagents, pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Integ / integ-ts (push) Has been cancelled
Integ / integ-fuse (push) Has been cancelled
Test (Install) / python-extra (databricks, mirage.resource.databricks_volume) (push) Has been cancelled
Test (Install) / python-extra (deepagents, mirage.agents.langchain) (push) Has been cancelled
Test (Install) / python-extra (email, mirage.resource.email) (push) Has been cancelled
Test (Install) / python-extra (fuse, mirage.fuse.mount) (push) Has been cancelled
Test (Install) / python-extra (hdf5, mirage.core.filetype.hdf5) (push) Has been cancelled
Test (Install) / python-extra (hf, mirage.resource.hf_buckets) (push) Has been cancelled
Test (Install) / python-extra (lancedb, mirage.resource.lancedb) (push) Has been cancelled
Test (Install) / python-extra (langfuse, mirage.resource.langfuse) (push) Has been cancelled
Test (Install) / python-extra (mongodb, mirage.resource.mongodb) (push) Has been cancelled
Test (Install) / python-extra (nextcloud, mirage.resource.nextcloud) (push) Has been cancelled
Test (Install) / python-extra (openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Install) / python-extra (openhands, mirage.agents.openhands, 3.12) (push) Has been cancelled
Test (Install) / python-extra (parquet, mirage.core.filetype.parquet) (push) Has been cancelled
Test (Install) / python-extra (postgres, mirage.resource.postgres) (push) Has been cancelled
Test (Install) / python-extra (pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Test (Install) / python-extra (qdrant, mirage.resource.qdrant) (push) Has been cancelled
Test (Install) / python-extra (redis, mirage.resource.redis) (push) Has been cancelled
Test (Install) / python-extra (s3, mirage.resource.s3) (push) Has been cancelled
Test (Install) / python-extra (ssh, mirage.resource.ssh) (push) Has been cancelled
Test (Install) / ts-minimal (push) Has been cancelled
Integ / changes (push) Has been skipped
Pre-commit / pre-commit (push) Failing after 1s
CLI exit codes / changes (push) Has been skipped
Test (Install) / changes (push) Has been skipped
Test (Python) / changes (push) Has been skipped
Test (TypeScript) / changes (push) Has been skipped
CLI exit codes / cli-gate (push) Has been cancelled
Test (Install) / test-install-gate (push) Has been cancelled
Integ / integ-gate (push) Has been cancelled
Test (Python) / test-python-gate (push) Has been cancelled
Test (TypeScript) / test-typescript-gate (push) Has been cancelled
Test (Install) / python-minimal (3.12) (push) Has been cancelled
Test (Install) / python-minimal (3.11) (push) Has been cancelled
Test (Install) / python-extra (agno, mirage.agents.agno) (push) Has been cancelled
Test (Install) / python-extra (chroma, mirage.resource.chroma) (push) Has been cancelled
Test (Install) / python-extra (pdf, mirage.core.filetype.pdf) (push) Has been cancelled
Integ / integ (push) Has been cancelled
Integ / integ-database (push) Has been cancelled
Integ / integ-database-ts (push) Has been cancelled
Integ / integ-data (push) Has been cancelled
Integ / integ-ssh (push) Has been cancelled
Integ / integ-ssh-ts (push) Has been cancelled
Test (Python) / audit (push) Has been cancelled
Test (TypeScript) / test (push) Has been cancelled
Test (TypeScript) / python-fs-shim (push) Has been cancelled
CLI exit codes / Python CLI (push) Has been cancelled
CLI exit codes / TypeScript CLI (push) Has been cancelled
CLI exit codes / Cross-language snapshot interop (push) Has been cancelled
Test (Python) / test (push) Has been cancelled
Test (Python) / import-isolation (deepagents, openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Python) / import-isolation (deepagents, pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Integ / integ-ts (push) Has been cancelled
Integ / integ-fuse (push) Has been cancelled
Test (Install) / python-extra (databricks, mirage.resource.databricks_volume) (push) Has been cancelled
Test (Install) / python-extra (deepagents, mirage.agents.langchain) (push) Has been cancelled
Test (Install) / python-extra (email, mirage.resource.email) (push) Has been cancelled
Test (Install) / python-extra (fuse, mirage.fuse.mount) (push) Has been cancelled
Test (Install) / python-extra (hdf5, mirage.core.filetype.hdf5) (push) Has been cancelled
Test (Install) / python-extra (hf, mirage.resource.hf_buckets) (push) Has been cancelled
Test (Install) / python-extra (lancedb, mirage.resource.lancedb) (push) Has been cancelled
Test (Install) / python-extra (langfuse, mirage.resource.langfuse) (push) Has been cancelled
Test (Install) / python-extra (mongodb, mirage.resource.mongodb) (push) Has been cancelled
Test (Install) / python-extra (nextcloud, mirage.resource.nextcloud) (push) Has been cancelled
Test (Install) / python-extra (openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Install) / python-extra (openhands, mirage.agents.openhands, 3.12) (push) Has been cancelled
Test (Install) / python-extra (parquet, mirage.core.filetype.parquet) (push) Has been cancelled
Test (Install) / python-extra (postgres, mirage.resource.postgres) (push) Has been cancelled
Test (Install) / python-extra (pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Test (Install) / python-extra (qdrant, mirage.resource.qdrant) (push) Has been cancelled
Test (Install) / python-extra (redis, mirage.resource.redis) (push) Has been cancelled
Test (Install) / python-extra (s3, mirage.resource.s3) (push) Has been cancelled
Test (Install) / python-extra (ssh, mirage.resource.ssh) (push) Has been cancelled
Test (Install) / ts-minimal (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,91 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.io.async_line_iterator import AsyncLineIterator
|
||||
|
||||
|
||||
async def _chunks(parts: list[bytes]):
|
||||
for p in parts:
|
||||
yield p
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clean_boundaries():
|
||||
source = _chunks([b"hello\nworld\n"])
|
||||
lines = [line async for line in AsyncLineIterator(source)]
|
||||
assert lines == [b"hello", b"world"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_split_across_chunks():
|
||||
source = _chunks([b"hel", b"lo\nwor", b"ld\n"])
|
||||
lines = [line async for line in AsyncLineIterator(source)]
|
||||
assert lines == [b"hello", b"world"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_trailing_newline():
|
||||
source = _chunks([b"hello\nworld"])
|
||||
lines = [line async for line in AsyncLineIterator(source)]
|
||||
assert lines == [b"hello", b"world"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_input():
|
||||
source = _chunks([])
|
||||
lines = [line async for line in AsyncLineIterator(source)]
|
||||
assert lines == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_chunk():
|
||||
source = _chunks([b"", b"hello\n", b""])
|
||||
lines = [line async for line in AsyncLineIterator(source)]
|
||||
assert lines == [b"hello"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_large_line():
|
||||
big = b"x" * 100000 + b"\n"
|
||||
source = _chunks([big[:8192], big[8192:]])
|
||||
lines = [line async for line in AsyncLineIterator(source)]
|
||||
assert lines == [b"x" * 100000]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_many_lines_one_chunk():
|
||||
source = _chunks([b"a\nb\nc\nd\n"])
|
||||
lines = [line async for line in AsyncLineIterator(source)]
|
||||
assert lines == [b"a", b"b", b"c", b"d"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_early_termination():
|
||||
pull_count = 0
|
||||
|
||||
async def _counting_chunks():
|
||||
nonlocal pull_count
|
||||
for i in range(1000):
|
||||
pull_count += 1
|
||||
yield f"line{i}\n".encode()
|
||||
|
||||
lines = []
|
||||
async for line in AsyncLineIterator(_counting_chunks()):
|
||||
lines.append(line)
|
||||
if len(lines) >= 3:
|
||||
break
|
||||
assert len(lines) == 3
|
||||
assert pull_count < 10
|
||||
@@ -0,0 +1,119 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
|
||||
from mirage.io.cachable_iterator import CachableAsyncIterator
|
||||
|
||||
|
||||
async def _async_source_two_chunks():
|
||||
yield b"aaa"
|
||||
yield b"bbb"
|
||||
|
||||
|
||||
async def _async_source_three_chunks():
|
||||
yield b"aaa"
|
||||
yield b"bbb"
|
||||
yield b"ccc"
|
||||
|
||||
|
||||
async def _run_yields_chunks():
|
||||
ci = CachableAsyncIterator(_async_source_two_chunks())
|
||||
chunks = []
|
||||
async for chunk in ci:
|
||||
chunks.append(chunk)
|
||||
assert chunks == [b"aaa", b"bbb"]
|
||||
|
||||
|
||||
async def _run_drain_after_partial():
|
||||
ci = CachableAsyncIterator(_async_source_three_chunks())
|
||||
chunk = await ci.__anext__()
|
||||
assert chunk == b"aaa"
|
||||
result = await ci.drain()
|
||||
assert result == b"aaabbbccc"
|
||||
|
||||
|
||||
async def _run_drain_without_iteration():
|
||||
ci = CachableAsyncIterator(_async_source_two_chunks())
|
||||
assert await ci.drain() == b"aaabbb"
|
||||
|
||||
|
||||
def test_cachable_async_iterator_yields_chunks():
|
||||
asyncio.run(_run_yields_chunks())
|
||||
|
||||
|
||||
def test_cachable_async_iterator_drain_after_partial():
|
||||
asyncio.run(_run_drain_after_partial())
|
||||
|
||||
|
||||
def test_cachable_async_iterator_drain_without_iteration():
|
||||
asyncio.run(_run_drain_without_iteration())
|
||||
|
||||
|
||||
async def _run_exhausted_false_before_full_consumption():
|
||||
ci = CachableAsyncIterator(_async_source_two_chunks())
|
||||
assert ci.exhausted is False
|
||||
await ci.__anext__()
|
||||
assert ci.exhausted is False
|
||||
|
||||
|
||||
async def _run_exhausted_true_after_full_iteration():
|
||||
ci = CachableAsyncIterator(_async_source_two_chunks())
|
||||
async for _ in ci:
|
||||
pass
|
||||
assert ci.exhausted is True
|
||||
|
||||
|
||||
async def _run_exhausted_true_after_drain():
|
||||
ci = CachableAsyncIterator(_async_source_two_chunks())
|
||||
await ci.__anext__()
|
||||
await ci.drain()
|
||||
assert ci.exhausted is True
|
||||
|
||||
|
||||
async def _run_drain_includes_already_consumed():
|
||||
ci = CachableAsyncIterator(_async_source_three_chunks())
|
||||
await ci.__anext__()
|
||||
await ci.__anext__()
|
||||
result = await ci.drain()
|
||||
assert result == b"aaabbbccc"
|
||||
|
||||
|
||||
async def _slow_source():
|
||||
yield b"aaa"
|
||||
await asyncio.sleep(0.05)
|
||||
yield b"bbb"
|
||||
await asyncio.sleep(0.05)
|
||||
yield b"ccc"
|
||||
|
||||
|
||||
def test_cachable_async_iterator_exhausted_false_before_full():
|
||||
asyncio.run(_run_exhausted_false_before_full_consumption())
|
||||
|
||||
|
||||
def test_cachable_async_iterator_exhausted_true_after_iteration():
|
||||
asyncio.run(_run_exhausted_true_after_full_iteration())
|
||||
|
||||
|
||||
def test_cachable_async_iterator_exhausted_true_after_drain():
|
||||
asyncio.run(_run_exhausted_true_after_drain())
|
||||
|
||||
|
||||
def test_cachable_async_iterator_drain_includes_already_consumed():
|
||||
asyncio.run(_run_drain_includes_already_consumed())
|
||||
|
||||
|
||||
async def _failing_source():
|
||||
yield b"aaa"
|
||||
raise RuntimeError("source failed")
|
||||
@@ -0,0 +1,74 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
|
||||
from mirage.io.types import IOResult
|
||||
|
||||
|
||||
def test_default_exit_code():
|
||||
io = IOResult()
|
||||
assert io.exit_code == 0
|
||||
|
||||
|
||||
def test_merge_combines_stderr():
|
||||
|
||||
async def _run():
|
||||
a = IOResult(stderr=b"err1")
|
||||
b = IOResult(stderr=b"err2")
|
||||
merged = await a.merge(b)
|
||||
assert merged.stderr == b"err1err2"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_merge_combines_cache():
|
||||
|
||||
async def _run():
|
||||
a = IOResult(cache=["/a"])
|
||||
b = IOResult(cache=["/b"])
|
||||
merged = await a.merge(b)
|
||||
assert merged.cache == ["/a", "/b"]
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_explicit_exit_code_clears_stream_source_issue_43():
|
||||
|
||||
async def _run():
|
||||
inner = IOResult(exit_code=1)
|
||||
outer = await IOResult().merge(inner)
|
||||
assert outer._stream_source is inner
|
||||
outer.exit_code = 0
|
||||
assert outer._stream_source is None
|
||||
outer.sync_exit_code()
|
||||
assert outer.exit_code == 0
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_explicit_exit_code_survives_chain_with_failing_leaf_issue_43():
|
||||
|
||||
async def _run():
|
||||
a = IOResult(exit_code=0)
|
||||
b = IOResult(exit_code=1)
|
||||
c = IOResult(exit_code=1)
|
||||
merged = await IOResult().merge(a)
|
||||
merged = await merged.merge(b)
|
||||
merged = await merged.merge(c)
|
||||
merged.exit_code = 0
|
||||
merged.sync_exit_code()
|
||||
assert merged.exit_code == 0
|
||||
|
||||
asyncio.run(_run())
|
||||
@@ -0,0 +1,345 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
|
||||
from mirage.io.stream import (async_chain, drain, exit_on_empty,
|
||||
merge_stdout_stderr, quiet_match)
|
||||
from mirage.io.types import IOResult
|
||||
|
||||
|
||||
async def _make_stream(*items):
|
||||
for item in items:
|
||||
yield item
|
||||
|
||||
|
||||
def test_exit_on_empty_with_items():
|
||||
|
||||
async def _run():
|
||||
io = IOResult()
|
||||
stream = exit_on_empty(_make_stream(b"a", b"b"), io)
|
||||
chunks = [chunk async for chunk in stream]
|
||||
assert chunks == [b"a", b"b"]
|
||||
assert io.exit_code == 0
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_exit_on_empty_no_items():
|
||||
|
||||
async def _run():
|
||||
io = IOResult()
|
||||
stream = exit_on_empty(_make_stream(), io)
|
||||
chunks = [chunk async for chunk in stream]
|
||||
assert chunks == []
|
||||
assert io.exit_code == 1
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_exit_on_empty_single_item():
|
||||
|
||||
async def _run():
|
||||
io = IOResult()
|
||||
stream = exit_on_empty(_make_stream(b"only"), io)
|
||||
chunks = [chunk async for chunk in stream]
|
||||
assert chunks == [b"only"]
|
||||
assert io.exit_code == 0
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_quiet_match_with_items():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(exit_code=1)
|
||||
stream = quiet_match(_make_stream(b"a", b"b"), io)
|
||||
chunks = [chunk async for chunk in stream]
|
||||
assert chunks == []
|
||||
assert io.exit_code == 0
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_drain_consumes_without_accumulating():
|
||||
|
||||
async def run():
|
||||
stream = _make_stream(b"hello", b"world")
|
||||
await drain(stream)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_drain_none():
|
||||
|
||||
async def run():
|
||||
await drain(None)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_drain_bytes():
|
||||
|
||||
async def run():
|
||||
await drain(b"hello")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_async_chain_two_streams():
|
||||
|
||||
async def run():
|
||||
a = _make_stream(b"hello ")
|
||||
b = _make_stream(b"world")
|
||||
chunks = []
|
||||
async for chunk in async_chain(a, b):
|
||||
chunks.append(chunk)
|
||||
assert b"".join(chunks) == b"hello world"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_async_chain_with_none():
|
||||
|
||||
async def run():
|
||||
a = None
|
||||
b = _make_stream(b"world")
|
||||
chunks = []
|
||||
async for chunk in async_chain(a, b):
|
||||
chunks.append(chunk)
|
||||
assert b"".join(chunks) == b"world"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_async_chain_with_bytes():
|
||||
|
||||
async def run():
|
||||
a = b"hello "
|
||||
b = b"world"
|
||||
chunks = []
|
||||
async for chunk in async_chain(a, b):
|
||||
chunks.append(chunk)
|
||||
assert b"".join(chunks) == b"hello world"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_async_chain_empty():
|
||||
|
||||
async def run():
|
||||
chunks = []
|
||||
async for chunk in async_chain(None, None):
|
||||
chunks.append(chunk)
|
||||
assert chunks == []
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_quiet_match_no_items():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(exit_code=1)
|
||||
stream = quiet_match(_make_stream(), io)
|
||||
chunks = [chunk async for chunk in stream]
|
||||
assert chunks == []
|
||||
assert io.exit_code == 1
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_merge_stdout_stderr_emits_stderr_first():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stderr=b"warn: bad\n")
|
||||
merged = merge_stdout_stderr(_make_stream(b"out1\n", b"out2\n"), io)
|
||||
chunks = [chunk async for chunk in merged]
|
||||
assert chunks[0] == b"warn: bad\n"
|
||||
assert chunks[1:] == [b"out1\n", b"out2\n"]
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_merge_stdout_stderr_clears_io_stderr():
|
||||
"""After merge, io.stderr is cleared so the pipeline accumulator
|
||||
does not double-emit it as pipeline stderr.
|
||||
"""
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stderr=b"err\n")
|
||||
merged = merge_stdout_stderr(_make_stream(b"x"), io)
|
||||
async for _ in merged:
|
||||
pass
|
||||
assert io.stderr is None
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_merge_stdout_stderr_streams_stdout_lazy():
|
||||
"""Stdout chunks pass through one at a time, never materialized."""
|
||||
pulls = 0
|
||||
|
||||
async def _lazy(n):
|
||||
nonlocal pulls
|
||||
for i in range(n):
|
||||
pulls += 1
|
||||
yield f"chunk{i}\n".encode()
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stderr=b"hi\n")
|
||||
merged = merge_stdout_stderr(_lazy(1000), io)
|
||||
seen = 0
|
||||
async for _ in merged:
|
||||
seen += 1
|
||||
if seen >= 5:
|
||||
break
|
||||
# 5 stdout chunks + 1 stderr blob = 6 yields; producer pulled
|
||||
# at most ~5 times, not 1000.
|
||||
assert pulls < 50, f"expected lazy pulls (~5), got {pulls}"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_merge_stdout_stderr_no_stderr():
|
||||
"""No stderr → just streams stdout."""
|
||||
|
||||
async def _run():
|
||||
io = IOResult()
|
||||
merged = merge_stdout_stderr(_make_stream(b"a", b"b"), io)
|
||||
chunks = [chunk async for chunk in merged]
|
||||
assert chunks == [b"a", b"b"]
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_merge_stdout_stderr_bytes_stdout():
|
||||
"""stdout as bytes (not iterator) still works."""
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stderr=b"e\n")
|
||||
merged = merge_stdout_stderr(b"out\n", io)
|
||||
chunks = [chunk async for chunk in merged]
|
||||
assert chunks == [b"e\n", b"out\n"]
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_close_quietly_fires_finally():
|
||||
"""Explicit aclose runs the producer's finally promptly."""
|
||||
from mirage.io.stream import close_quietly
|
||||
closed = []
|
||||
|
||||
async def producer():
|
||||
try:
|
||||
for i in range(100):
|
||||
yield f"chunk{i}\n".encode()
|
||||
finally:
|
||||
closed.append("done")
|
||||
|
||||
async def _run():
|
||||
p = producer()
|
||||
async for _ in p:
|
||||
break
|
||||
assert closed == [], "finally fires only on close"
|
||||
await close_quietly(p)
|
||||
assert closed == ["done"], "finally fires after explicit close"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_close_quietly_safe_on_bytes_and_none():
|
||||
"""close_quietly is harmless on non-iterator inputs."""
|
||||
from mirage.io.stream import close_quietly
|
||||
|
||||
async def _run():
|
||||
await close_quietly(None)
|
||||
await close_quietly(b"some bytes")
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_close_quietly_swallows_exceptions():
|
||||
"""A broken aclose impl shouldn't propagate."""
|
||||
from mirage.io.stream import close_quietly
|
||||
|
||||
class Bad:
|
||||
|
||||
async def aclose(self):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
async def _run():
|
||||
await close_quietly(Bad()) # should not raise
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_chain_cachables_live_pull_in_order():
|
||||
from mirage.io.cachable_iterator import CachableAsyncIterator
|
||||
from mirage.io.stream import chain_cachables
|
||||
|
||||
async def _run():
|
||||
a = CachableAsyncIterator(_make_stream(b"a1", b"a2"))
|
||||
b = CachableAsyncIterator(_make_stream(b"b1"))
|
||||
chunks = [c async for c in chain_cachables(a, b)]
|
||||
assert chunks == [b"a1", b"a2", b"b1"]
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_chain_cachables_replays_drained_iterator():
|
||||
from mirage.io.cachable_iterator import CachableAsyncIterator
|
||||
from mirage.io.stream import chain_cachables
|
||||
|
||||
async def _run():
|
||||
a = CachableAsyncIterator(_make_stream(b"a1", b"a2"))
|
||||
b = CachableAsyncIterator(_make_stream(b"b1"))
|
||||
assert await a.drain() == b"a1a2"
|
||||
assert await b.drain() == b"b1"
|
||||
chunks = [c async for c in chain_cachables(a, b)]
|
||||
assert chunks == [b"a1", b"a2", b"b1"]
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_chain_cachables_replays_partial_then_drained():
|
||||
from mirage.io.cachable_iterator import CachableAsyncIterator
|
||||
from mirage.io.stream import chain_cachables
|
||||
|
||||
async def _run():
|
||||
a = CachableAsyncIterator(_make_stream(b"a1", b"a2", b"a3"))
|
||||
b = CachableAsyncIterator(_make_stream(b"b1"))
|
||||
chain = chain_cachables(a, b)
|
||||
assert await chain.__anext__() == b"a1"
|
||||
await a.drain()
|
||||
await b.drain()
|
||||
rest = [c async for c in chain]
|
||||
assert rest == [b"a2", b"a3", b"b1"]
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_chain_cachables_early_stop_leaves_later_untouched():
|
||||
from mirage.io.cachable_iterator import CachableAsyncIterator
|
||||
from mirage.io.stream import chain_cachables
|
||||
|
||||
async def _run():
|
||||
a = CachableAsyncIterator(_make_stream(b"a1", b"a2"))
|
||||
b = CachableAsyncIterator(_make_stream(b"b1"))
|
||||
chain = chain_cachables(a, b)
|
||||
assert await chain.__anext__() == b"a1"
|
||||
await chain.aclose()
|
||||
assert b.buffered_chunks == []
|
||||
assert not b.exhausted
|
||||
|
||||
asyncio.run(_run())
|
||||
@@ -0,0 +1,86 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
|
||||
from mirage.io.sync_bridge import async_to_sync_iter
|
||||
|
||||
|
||||
async def _async_chunks():
|
||||
yield b"aaa"
|
||||
yield b"bbb"
|
||||
yield b"ccc"
|
||||
|
||||
|
||||
async def _async_single_chunk():
|
||||
yield b"only"
|
||||
|
||||
|
||||
async def _async_empty():
|
||||
return
|
||||
yield b""
|
||||
|
||||
|
||||
def _make_loop() -> asyncio.AbstractEventLoop:
|
||||
loop = asyncio.new_event_loop()
|
||||
return loop
|
||||
|
||||
|
||||
def test_basic_conversion():
|
||||
loop = _make_loop()
|
||||
try:
|
||||
chunks = list(async_to_sync_iter(_async_chunks(), loop))
|
||||
assert chunks == [b"aaa", b"bbb", b"ccc"]
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
def test_concatenated_output():
|
||||
loop = _make_loop()
|
||||
try:
|
||||
result = b"".join(async_to_sync_iter(_async_chunks(), loop))
|
||||
assert result == b"aaabbbccc"
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
def test_single_chunk():
|
||||
loop = _make_loop()
|
||||
try:
|
||||
chunks = list(async_to_sync_iter(_async_single_chunk(), loop))
|
||||
assert chunks == [b"only"]
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
def test_empty_iterator():
|
||||
loop = _make_loop()
|
||||
try:
|
||||
chunks = list(async_to_sync_iter(_async_empty(), loop))
|
||||
assert chunks == []
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
def test_early_termination():
|
||||
loop = _make_loop()
|
||||
consumed = []
|
||||
try:
|
||||
it = async_to_sync_iter(_async_chunks(), loop)
|
||||
first = next(it)
|
||||
consumed.append(first)
|
||||
assert first == b"aaa"
|
||||
assert len(consumed) == 1
|
||||
finally:
|
||||
loop.close()
|
||||
@@ -0,0 +1,208 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
|
||||
from mirage.io.cachable_iterator import CachableAsyncIterator
|
||||
from mirage.io.types import IOResult
|
||||
|
||||
|
||||
async def _async_source(*chunks):
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
def test_ioresult_reads_accepts_bytes():
|
||||
io = IOResult(reads={"/a": b"hello"})
|
||||
assert io.reads["/a"] == b"hello"
|
||||
|
||||
|
||||
def test_ioresult_reads_accepts_async_iterator():
|
||||
ait = _async_source(b"chunk")
|
||||
io = IOResult(reads={"/a": ait})
|
||||
assert io.reads["/a"] is ait
|
||||
|
||||
|
||||
def test_ioresult_cache_default_empty():
|
||||
io = IOResult()
|
||||
assert io.cache == []
|
||||
|
||||
|
||||
def test_ioresult_cache_set():
|
||||
io = IOResult(
|
||||
reads={"/a": _async_source(b"x")},
|
||||
cache=["/a"],
|
||||
)
|
||||
assert io.cache == ["/a"]
|
||||
|
||||
|
||||
def test_ioresult_merge_combines_cache():
|
||||
|
||||
async def _run():
|
||||
left = IOResult(cache=["/a"])
|
||||
right = IOResult(cache=["/b"])
|
||||
merged = await left.merge(right)
|
||||
assert merged.cache == ["/a", "/b"]
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_merge_mixed_reads():
|
||||
|
||||
async def _run():
|
||||
left = IOResult(reads={"/a": b"hello"})
|
||||
ait = _async_source(b"x")
|
||||
right = IOResult(reads={"/b": ait})
|
||||
merged = await left.merge(right)
|
||||
assert merged.reads["/a"] == b"hello"
|
||||
assert merged.reads["/b"] is ait
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_stdout_defaults_none():
|
||||
io = IOResult()
|
||||
assert io.stdout is None
|
||||
assert io.stderr is None
|
||||
assert io.exit_code == 0
|
||||
|
||||
|
||||
def test_ioresult_materialize_stdout_bytes():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stdout=b"hello")
|
||||
assert await io.materialize_stdout() == b"hello"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_materialize_stdout_exhausted_async():
|
||||
|
||||
async def _run():
|
||||
ci = CachableAsyncIterator(_async_source(b"he", b"llo"))
|
||||
await ci.drain()
|
||||
io = IOResult(stdout=ci)
|
||||
assert await io.materialize_stdout() == b"hello"
|
||||
assert io.stdout == b"hello"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_materialize_stdout_none():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stdout=None)
|
||||
assert await io.materialize_stdout() == b""
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_stdout_str():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stdout=b"hello world")
|
||||
assert await io.stdout_str() == "hello world"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_materialize_stderr():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stderr=b"error msg")
|
||||
assert await io.materialize_stderr() == b"error msg"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_stderr_str():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stderr=b"error")
|
||||
assert await io.stderr_str() == "error"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_merge_stdout_takes_right():
|
||||
|
||||
async def _run():
|
||||
left = IOResult(stdout=b"left")
|
||||
right = IOResult(stdout=b"right")
|
||||
merged = await left.merge(right)
|
||||
assert await merged.materialize_stdout() == b"right"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_merge_exit_code_takes_right():
|
||||
|
||||
async def _run():
|
||||
left = IOResult(exit_code=0)
|
||||
right = IOResult(exit_code=1)
|
||||
merged = await left.merge(right)
|
||||
assert merged.exit_code == 1
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_merge_stderr_concatenates():
|
||||
|
||||
async def _run():
|
||||
left = IOResult(stderr=b"err1 ")
|
||||
right = IOResult(stderr=b"err2")
|
||||
merged = await left.merge(right)
|
||||
assert await merged.materialize_stderr() == b"err1 err2"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_ioresult_merge_stderr_none_both():
|
||||
|
||||
async def _run():
|
||||
left = IOResult(stderr=None)
|
||||
right = IOResult(stderr=None)
|
||||
merged = await left.merge(right)
|
||||
assert merged.stderr is None
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_stdout_str_materializes_async_iterator():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stdout=_async_source(b"hel", b"lo"))
|
||||
assert await io.stdout_str() == "hello"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_stderr_str_materializes_async_iterator():
|
||||
|
||||
async def _run():
|
||||
io = IOResult(stderr=_async_source(b"err", b"or"))
|
||||
assert await io.stderr_str() == "error"
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_merge_materializes_async_stderr():
|
||||
|
||||
async def _run():
|
||||
left = IOResult(stderr=_async_source(b"warn"))
|
||||
right = IOResult(stdout=b"out")
|
||||
merged = await left.merge(right)
|
||||
assert await merged.stderr_str() == "warn"
|
||||
|
||||
asyncio.run(_run())
|
||||
Reference in New Issue
Block a user