chore: import upstream snapshot with attribution
Integ / changes (push) Has been skipped
Pre-commit / pre-commit (push) Failing after 1s
CLI exit codes / changes (push) Has been skipped
Test (Install) / changes (push) Has been skipped
Test (Python) / changes (push) Has been skipped
Test (TypeScript) / changes (push) Has been skipped
CLI exit codes / cli-gate (push) Has been cancelled
Test (Install) / test-install-gate (push) Has been cancelled
Integ / integ-gate (push) Has been cancelled
Test (Python) / test-python-gate (push) Has been cancelled
Test (TypeScript) / test-typescript-gate (push) Has been cancelled
Test (Install) / python-minimal (3.12) (push) Has been cancelled
Test (Install) / python-minimal (3.11) (push) Has been cancelled
Test (Install) / python-extra (agno, mirage.agents.agno) (push) Has been cancelled
Test (Install) / python-extra (chroma, mirage.resource.chroma) (push) Has been cancelled
Test (Install) / python-extra (pdf, mirage.core.filetype.pdf) (push) Has been cancelled
Integ / integ (push) Has been cancelled
Integ / integ-database (push) Has been cancelled
Integ / integ-database-ts (push) Has been cancelled
Integ / integ-data (push) Has been cancelled
Integ / integ-ssh (push) Has been cancelled
Integ / integ-ssh-ts (push) Has been cancelled
Test (Python) / audit (push) Has been cancelled
Test (TypeScript) / test (push) Has been cancelled
Test (TypeScript) / python-fs-shim (push) Has been cancelled
CLI exit codes / Python CLI (push) Has been cancelled
CLI exit codes / TypeScript CLI (push) Has been cancelled
CLI exit codes / Cross-language snapshot interop (push) Has been cancelled
Test (Python) / test (push) Has been cancelled
Test (Python) / import-isolation (deepagents, openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Python) / import-isolation (deepagents, pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Integ / integ-ts (push) Has been cancelled
Integ / integ-fuse (push) Has been cancelled
Test (Install) / python-extra (databricks, mirage.resource.databricks_volume) (push) Has been cancelled
Test (Install) / python-extra (deepagents, mirage.agents.langchain) (push) Has been cancelled
Test (Install) / python-extra (email, mirage.resource.email) (push) Has been cancelled
Test (Install) / python-extra (fuse, mirage.fuse.mount) (push) Has been cancelled
Test (Install) / python-extra (hdf5, mirage.core.filetype.hdf5) (push) Has been cancelled
Test (Install) / python-extra (hf, mirage.resource.hf_buckets) (push) Has been cancelled
Test (Install) / python-extra (lancedb, mirage.resource.lancedb) (push) Has been cancelled
Test (Install) / python-extra (langfuse, mirage.resource.langfuse) (push) Has been cancelled
Test (Install) / python-extra (mongodb, mirage.resource.mongodb) (push) Has been cancelled
Test (Install) / python-extra (nextcloud, mirage.resource.nextcloud) (push) Has been cancelled
Test (Install) / python-extra (openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Install) / python-extra (openhands, mirage.agents.openhands, 3.12) (push) Has been cancelled
Test (Install) / python-extra (parquet, mirage.core.filetype.parquet) (push) Has been cancelled
Test (Install) / python-extra (postgres, mirage.resource.postgres) (push) Has been cancelled
Test (Install) / python-extra (pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Test (Install) / python-extra (qdrant, mirage.resource.qdrant) (push) Has been cancelled
Test (Install) / python-extra (redis, mirage.resource.redis) (push) Has been cancelled
Test (Install) / python-extra (s3, mirage.resource.s3) (push) Has been cancelled
Test (Install) / python-extra (ssh, mirage.resource.ssh) (push) Has been cancelled
Test (Install) / ts-minimal (push) Has been cancelled
Integ / changes (push) Has been skipped
Pre-commit / pre-commit (push) Failing after 1s
CLI exit codes / changes (push) Has been skipped
Test (Install) / changes (push) Has been skipped
Test (Python) / changes (push) Has been skipped
Test (TypeScript) / changes (push) Has been skipped
CLI exit codes / cli-gate (push) Has been cancelled
Test (Install) / test-install-gate (push) Has been cancelled
Integ / integ-gate (push) Has been cancelled
Test (Python) / test-python-gate (push) Has been cancelled
Test (TypeScript) / test-typescript-gate (push) Has been cancelled
Test (Install) / python-minimal (3.12) (push) Has been cancelled
Test (Install) / python-minimal (3.11) (push) Has been cancelled
Test (Install) / python-extra (agno, mirage.agents.agno) (push) Has been cancelled
Test (Install) / python-extra (chroma, mirage.resource.chroma) (push) Has been cancelled
Test (Install) / python-extra (pdf, mirage.core.filetype.pdf) (push) Has been cancelled
Integ / integ (push) Has been cancelled
Integ / integ-database (push) Has been cancelled
Integ / integ-database-ts (push) Has been cancelled
Integ / integ-data (push) Has been cancelled
Integ / integ-ssh (push) Has been cancelled
Integ / integ-ssh-ts (push) Has been cancelled
Test (Python) / audit (push) Has been cancelled
Test (TypeScript) / test (push) Has been cancelled
Test (TypeScript) / python-fs-shim (push) Has been cancelled
CLI exit codes / Python CLI (push) Has been cancelled
CLI exit codes / TypeScript CLI (push) Has been cancelled
CLI exit codes / Cross-language snapshot interop (push) Has been cancelled
Test (Python) / test (push) Has been cancelled
Test (Python) / import-isolation (deepagents, openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Python) / import-isolation (deepagents, pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Integ / integ-ts (push) Has been cancelled
Integ / integ-fuse (push) Has been cancelled
Test (Install) / python-extra (databricks, mirage.resource.databricks_volume) (push) Has been cancelled
Test (Install) / python-extra (deepagents, mirage.agents.langchain) (push) Has been cancelled
Test (Install) / python-extra (email, mirage.resource.email) (push) Has been cancelled
Test (Install) / python-extra (fuse, mirage.fuse.mount) (push) Has been cancelled
Test (Install) / python-extra (hdf5, mirage.core.filetype.hdf5) (push) Has been cancelled
Test (Install) / python-extra (hf, mirage.resource.hf_buckets) (push) Has been cancelled
Test (Install) / python-extra (lancedb, mirage.resource.lancedb) (push) Has been cancelled
Test (Install) / python-extra (langfuse, mirage.resource.langfuse) (push) Has been cancelled
Test (Install) / python-extra (mongodb, mirage.resource.mongodb) (push) Has been cancelled
Test (Install) / python-extra (nextcloud, mirage.resource.nextcloud) (push) Has been cancelled
Test (Install) / python-extra (openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Install) / python-extra (openhands, mirage.agents.openhands, 3.12) (push) Has been cancelled
Test (Install) / python-extra (parquet, mirage.core.filetype.parquet) (push) Has been cancelled
Test (Install) / python-extra (postgres, mirage.resource.postgres) (push) Has been cancelled
Test (Install) / python-extra (pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Test (Install) / python-extra (qdrant, mirage.resource.qdrant) (push) Has been cancelled
Test (Install) / python-extra (redis, mirage.resource.redis) (push) Has been cancelled
Test (Install) / python-extra (s3, mirage.resource.s3) (push) Has been cancelled
Test (Install) / python-extra (ssh, mirage.resource.ssh) (push) Has been cancelled
Test (Install) / ts-minimal (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,398 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from mirage import MountMode, Workspace
|
||||
from mirage.resource.gcs import GCSConfig, GCSResource
|
||||
|
||||
load_dotenv(".env.development")
|
||||
|
||||
config = GCSConfig(
|
||||
bucket=os.environ["GCS_BUCKET"],
|
||||
access_key_id=os.environ["GCS_ACCESS_KEY_ID"],
|
||||
secret_access_key=os.environ["GCS_SECRET_ACCESS_KEY"],
|
||||
)
|
||||
|
||||
resource = GCSResource(config)
|
||||
ws = Workspace({"/gcs/": resource}, mode=MountMode.READ)
|
||||
|
||||
|
||||
def ops_summary() -> str:
|
||||
records = ws.ops.records
|
||||
total = sum(r.bytes for r in records)
|
||||
return f"{len(records)} ops, {total} bytes transferred"
|
||||
|
||||
|
||||
async def main():
|
||||
# ── discover structure ────────────────────────────
|
||||
print("=== ls /gcs/ ===")
|
||||
r = await ws.execute("ls /gcs/")
|
||||
print(await r.stdout_str())
|
||||
|
||||
print("=== ls /gcs/data/ ===")
|
||||
r = await ws.execute("ls /gcs/data/")
|
||||
print(await r.stdout_str())
|
||||
|
||||
# ── tree ──────────────────────────────────────────
|
||||
print("=== tree /gcs/ ===")
|
||||
r = await ws.execute("tree /gcs/")
|
||||
print(await r.stdout_str())
|
||||
|
||||
# ── stat (directory prefix + file) ──────────────────
|
||||
print("=== stat /gcs/data (directory prefix) ===")
|
||||
r = await ws.execute("stat /gcs/data")
|
||||
print(f" {(await r.stdout_str()).strip()}")
|
||||
|
||||
print("\n=== stat /gcs/data/example.json ===")
|
||||
r = await ws.execute("stat /gcs/data/example.json")
|
||||
print(f" {(await r.stdout_str()).strip()}")
|
||||
|
||||
# ── cat json ──────────────────────────────────────
|
||||
print("\n=== cat /gcs/data/example.json | head -n 10 ===")
|
||||
r = await ws.execute("cat /gcs/data/example.json | head -n 10")
|
||||
print(await r.stdout_str())
|
||||
|
||||
# ── head / tail on jsonl ──────────────────────────
|
||||
print("=== head -n 3 /gcs/data/example.jsonl ===")
|
||||
r = await ws.execute("head -n 3 /gcs/data/example.jsonl")
|
||||
print((await r.stdout_str())[:300])
|
||||
|
||||
print("\n=== tail -n 2 /gcs/data/example.jsonl ===")
|
||||
r = await ws.execute("tail -n 2 /gcs/data/example.jsonl")
|
||||
print((await r.stdout_str())[:300])
|
||||
|
||||
# ── wc ────────────────────────────────────────────
|
||||
print("\n=== wc -l /gcs/data/example.jsonl ===")
|
||||
r = await ws.execute("wc -l /gcs/data/example.jsonl")
|
||||
print(f" {(await r.stdout_str()).strip()}")
|
||||
|
||||
# ── grep ──────────────────────────────────────────
|
||||
print("\n=== grep -c mirage /gcs/data/example.jsonl ===")
|
||||
r = await ws.execute("grep -c mirage /gcs/data/example.jsonl")
|
||||
print(f" count: {(await r.stdout_str()).strip()}")
|
||||
|
||||
print("\n=== grep mirage /gcs/data/example.jsonl | head -n 3 ===")
|
||||
r = await ws.execute("grep mirage /gcs/data/example.jsonl | head -n 3")
|
||||
lines = (await r.stdout_str()).strip().splitlines()
|
||||
for ln in lines:
|
||||
print(f" {ln[:100]}...")
|
||||
|
||||
# ── find ──────────────────────────────────────────
|
||||
print("\n=== find /gcs/ -name '*.json' ===")
|
||||
r = await ws.execute("find /gcs/ -name '*.json'")
|
||||
print(await r.stdout_str())
|
||||
|
||||
print("=== find /gcs/ -name '*.parquet' ===")
|
||||
r = await ws.execute("find /gcs/ -name '*.parquet'")
|
||||
print(await r.stdout_str())
|
||||
|
||||
# ── jq ────────────────────────────────────────────
|
||||
print("=== jq .metadata /gcs/data/example.json ===")
|
||||
r = await ws.execute("jq .metadata /gcs/data/example.json")
|
||||
print(f" {(await r.stdout_str()).strip()[:200]}")
|
||||
|
||||
print("\n=== jq '.departments[].teams[].name'"
|
||||
" /gcs/data/example.json ===")
|
||||
r = await ws.execute(
|
||||
'jq ".departments[].teams[].name" /gcs/data/example.json')
|
||||
print(f" {(await r.stdout_str()).strip()}")
|
||||
|
||||
# ── pipelines ─────────────────────────────────────
|
||||
print("\n=== cat example.jsonl | grep queue-operation"
|
||||
" | sort | uniq | wc -l ===")
|
||||
r = await ws.execute("cat /gcs/data/example.jsonl"
|
||||
" | grep queue-operation | sort | uniq | wc -l")
|
||||
print(f" unique lines: {(await r.stdout_str()).strip()}")
|
||||
|
||||
# ── cd + relative paths ───────────────────────────
|
||||
print("\n=== pwd ===")
|
||||
r = await ws.execute("pwd")
|
||||
print(f" {(await r.stdout_str()).strip()}")
|
||||
|
||||
print('\n=== cd /gcs/data ===')
|
||||
r = await ws.execute("cd /gcs/data")
|
||||
print(f" exit={r.exit_code}")
|
||||
|
||||
print("\n=== pwd (after cd) ===")
|
||||
r = await ws.execute("pwd")
|
||||
print(f" {(await r.stdout_str()).strip()}")
|
||||
|
||||
print("\n=== ls (relative) ===")
|
||||
r = await ws.execute("ls")
|
||||
print(await r.stdout_str())
|
||||
|
||||
print("=== head -n 3 example.json (relative) ===")
|
||||
r = await ws.execute("head -n 3 example.json")
|
||||
print(await r.stdout_str())
|
||||
|
||||
# ── streaming & barrier scenarios ─────────────────
|
||||
print("\n=== cat | grep | head (streaming drain) ===")
|
||||
r = await ws.execute("cat /gcs/data/example.jsonl | grep queue | head -n 3"
|
||||
)
|
||||
lines = (await r.stdout_str()).strip().splitlines()
|
||||
print(f" got {len(lines)} lines (expected 3)")
|
||||
|
||||
print("\n=== grep -q && echo (barrier VALUE) ===")
|
||||
r = await ws.execute(
|
||||
'grep -q queue /gcs/data/example.jsonl && echo "found"')
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
print(f" exit: {r.exit_code}")
|
||||
|
||||
print("\n=== grep -q || echo (barrier OR) ===")
|
||||
r = await ws.execute('grep -q NONEXISTENT_STRING /gcs/data/example.jsonl'
|
||||
' || echo "not found"')
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
print(f" exit: {r.exit_code}")
|
||||
|
||||
print("\n=== grep ; grep (semicolon materialization) ===")
|
||||
r = await ws.execute("grep -c queue /gcs/data/example.jsonl"
|
||||
"; grep -c mirage /gcs/data/example.jsonl")
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
|
||||
print("\n=== grep missing ; echo $? (semicolon exit code) ===")
|
||||
r = await ws.execute("grep NONEXISTENT_STRING /gcs/data/example.jsonl"
|
||||
"; echo $?")
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
|
||||
print("\n=== cat nonexistent 2>&1 | head (stderr in pipe) ===")
|
||||
r = await ws.execute("cat /gcs/data/nonexistent_file 2>&1 | head -n 1")
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
print(f" exit: {r.exit_code}")
|
||||
|
||||
# ── Step 16: 2>&1 fixes — no double-emit, stream stdout ──
|
||||
# Without 2>&1: error stays in stderr, stdout empty.
|
||||
print("\n=== cat nonexistent | cat (no merge: error in stderr) ===")
|
||||
r = await ws.execute("cat /gcs/data/nonexistent_file | cat")
|
||||
print(f" stdout: '{(await r.stdout_str()).strip()}' (expect empty)")
|
||||
print(f" stderr: '{(await r.stderr_str()).strip()[:80]}'")
|
||||
|
||||
# With 2>&1: error goes into pipe → reaches final stdout.
|
||||
# Crucially, final stderr should be empty (no double-emit).
|
||||
print("\n=== cat nonexistent 2>&1 | cat (no double-emit) ===")
|
||||
r = await ws.execute("cat /gcs/data/nonexistent_file 2>&1 | cat")
|
||||
out = (await r.stdout_str()).strip()
|
||||
err = (await r.stderr_str()).strip()
|
||||
print(f" stdout: '{out[:80]}'")
|
||||
print(f" stderr: '{err}' (expect empty: no double-emit)")
|
||||
|
||||
# 2>&1 streams stdout chunk-by-chunk through merge.
|
||||
# Real GCS payload (5766 lines) piped through 2>&1 to wc -l
|
||||
# should match wc -l on the file directly.
|
||||
print("\n=== cat large 2>&1 | wc -l (streams real payload) ===")
|
||||
r = await ws.execute("wc -l /gcs/data/example.jsonl")
|
||||
expected = int((await r.stdout_str()).strip().split()[0])
|
||||
r = await ws.execute("cat /gcs/data/example.jsonl 2>&1 | wc -l")
|
||||
got = int((await r.stdout_str()).strip())
|
||||
print(f" expected: {expected} got: {got} "
|
||||
f"{'OK' if got == expected else 'MISMATCH'}")
|
||||
|
||||
print("\n=== cat | sort | uniq | wc -l (full pipeline) ===")
|
||||
r = await ws.execute("cat /gcs/data/example.jsonl"
|
||||
" | sort | uniq | wc -l")
|
||||
print(f" unique lines: {(await r.stdout_str()).strip()}")
|
||||
|
||||
# ── background job scenarios ────────────────────
|
||||
print("\n=== grep -c & echo kicked off; wait (bg job) ===")
|
||||
r = await ws.execute("grep -c queue /gcs/data/example.jsonl &"
|
||||
" echo 'kicked off'; wait")
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
|
||||
print("\n=== sleep 0 & cat (bg doesn't consume stdin) ===")
|
||||
r = await ws.execute("sleep 0 & cat /gcs/data/example.json | head -n 1")
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
|
||||
print("\n=== cat nonexistent & echo ok (bg error handled) ===")
|
||||
r = await ws.execute(
|
||||
"cat /gcs/data/nonexistent_file & echo ok; wait; echo done")
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
print(f" exit: {r.exit_code}")
|
||||
|
||||
print("\n=== multiple bg: grep & wc & wait (parallel) ===")
|
||||
r = await ws.execute("grep -c queue /gcs/data/example.jsonl &"
|
||||
" wc -l /gcs/data/example.jsonl &"
|
||||
" wait; echo all done")
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
|
||||
# ── lazy stdin in loops (Step 15) ────────────────
|
||||
# Functional checks. True laziness is measured by the unit test
|
||||
# (test_while_read_break_stops_pulling) — it counts producer pulls.
|
||||
# End-to-end over the network these are just correctness checks.
|
||||
|
||||
# Bounded loop (head limits upstream → exact iter count).
|
||||
# With lazy stdin, `head -n 5` triggers EOF after 5 lines, the
|
||||
# while loop sees readline()==None, exits cleanly.
|
||||
print("\n=== head -n 5 | while read; do echo (bounded loop) ===")
|
||||
r = await ws.execute("cat /gcs/data/example.jsonl | head -n 5"
|
||||
" | while read LINE; do echo got; done | wc -l")
|
||||
print(f" iterations: {(await r.stdout_str()).strip()} (expected 5)")
|
||||
|
||||
# Early break: only one iter, rest of upstream untouched.
|
||||
# Visible via unit test `test_while_read_break_stops_pulling`;
|
||||
# here we only check stdout shape.
|
||||
print("\n=== while read; break (early exit) ===")
|
||||
r = await ws.execute("cat /gcs/data/example.jsonl | head -n 100"
|
||||
" | while read LINE; do echo first; break; done")
|
||||
out = (await r.stdout_str()).strip().splitlines()
|
||||
print(f" stdout lines: {len(out)} (expected 1) exit={r.exit_code}")
|
||||
|
||||
# for-loop over fixed value list, with stdin available to body.
|
||||
# Each iter calls `read` once → consumes one line from buffer.
|
||||
print("\n=== for x in a b c; do read LINE (loop reads buffer) ===")
|
||||
r = await ws.execute(
|
||||
"cat /gcs/data/example.jsonl | head -n 3"
|
||||
" | for x in a b c; do read LINE; echo \"$x:${LINE:0:30}\"; done")
|
||||
for line in (await r.stdout_str()).strip().splitlines():
|
||||
print(f" {line}")
|
||||
|
||||
# Note: while-loop over an unbounded stream is currently capped
|
||||
# at _MAX_WHILE=10000 iterations regardless of input size. This is
|
||||
# a separate safety limit, not a laziness issue.
|
||||
print("\n (note: while over unbounded stream caps at 10000 iters)")
|
||||
|
||||
# ── quoting / escaping patterns commonly used by AI agents ──
|
||||
# These exercise the bash double-quote escape rules + variable
|
||||
# expansion semantics the agent relies on.
|
||||
|
||||
print("\n=== echo \"\\$X\" (escaped dollar stays literal) ===")
|
||||
await ws.execute("export X=expanded")
|
||||
r = await ws.execute('echo "\\$X"')
|
||||
print(f" stdout: {json.dumps((await r.stdout_str()).strip())}"
|
||||
" (expect '$X')")
|
||||
|
||||
print("\n=== echo \"$X\" (unescaped dollar expands) ===")
|
||||
r = await ws.execute('echo "$X"')
|
||||
print(f" stdout: {json.dumps((await r.stdout_str()).strip())}"
|
||||
" (expect 'expanded')")
|
||||
|
||||
print("\n=== echo '$X' (single quotes keep $X literal) ===")
|
||||
r = await ws.execute("echo '$X'")
|
||||
print(f" stdout: {json.dumps((await r.stdout_str()).strip())}"
|
||||
" (expect '$X')")
|
||||
|
||||
print("\n=== cat \"$DIR/example.json\" (env var in path) ===")
|
||||
await ws.execute("export DIR=/gcs/data")
|
||||
r = await ws.execute('cat "$DIR/example.json" | head -n 3')
|
||||
out = (await r.stdout_str()).strip().splitlines()
|
||||
print(f" first lines: {json.dumps(out, separators=(',', ':'))}")
|
||||
|
||||
print("\n=== cat $(echo /gcs/data/example.json) | head -n 1"
|
||||
" (command sub as path) ===")
|
||||
r = await ws.execute("cat $(echo /gcs/data/example.json) | head -n 1")
|
||||
print(f" stdout: {(await r.stdout_str()).strip()}")
|
||||
|
||||
print("\n=== grep \"$(echo queue)\" /gcs/data/example.jsonl"
|
||||
" | wc -l (sub as pattern) ===")
|
||||
r = await ws.execute('grep "$(echo queue)" /gcs/data/example.jsonl | wc -l'
|
||||
)
|
||||
print(f" count: {(await r.stdout_str()).strip()}")
|
||||
|
||||
# ── on-the-fly max_drain_bytes (cancellable cache drain) ──────────
|
||||
# Demonstrates: set a small budget → cat | head leaves the source
|
||||
# only partially read → background drain trips the budget → cache
|
||||
# is NOT populated. Then unset the budget → drain completes → cache
|
||||
# IS populated. Drain tasks are awaited explicitly (vs sleeping)
|
||||
# so we don't race the network.
|
||||
target = "/gcs/data/example.jsonl"
|
||||
await ws.cache.clear()
|
||||
print("\n=== max_drain_bytes=1MB then cat | head (drain cancelled) ===")
|
||||
ws.max_drain_bytes = 1_000_000
|
||||
r = await ws.execute(f"cat {target} | head -n 3")
|
||||
print(f" head returned {len((await r.stdout_str()).splitlines())} lines")
|
||||
drain_keys = list(ws.cache._drain_tasks.keys())
|
||||
print(f" drain tasks: {json.dumps(drain_keys, separators=(',', ':'))}")
|
||||
for k in drain_keys:
|
||||
await ws.cache._drain_tasks[k]
|
||||
cached = None
|
||||
for k in drain_keys:
|
||||
cached = await ws.cache.get(k) or cached
|
||||
if cached:
|
||||
status = f"POPULATED ({len(cached)} bytes)"
|
||||
else:
|
||||
status = "EMPTY (drain cancelled, as expected)"
|
||||
print(f" cache after small budget: {status}")
|
||||
|
||||
await ws.cache.clear()
|
||||
print("\n=== max_drain_bytes=None then cat | head (drain completes) ===")
|
||||
ws.max_drain_bytes = None
|
||||
r = await ws.execute(f"cat {target} | head -n 3")
|
||||
print(f" head returned {len((await r.stdout_str()).splitlines())} lines")
|
||||
drain_keys = list(ws.cache._drain_tasks.keys())
|
||||
print(f" drain tasks: {json.dumps(drain_keys, separators=(',', ':'))}")
|
||||
for k in drain_keys:
|
||||
await ws.cache._drain_tasks[k]
|
||||
cached = None
|
||||
for k in drain_keys:
|
||||
cached = await ws.cache.get(k) or cached
|
||||
if cached:
|
||||
status = f"POPULATED ({len(cached)} bytes, as expected)"
|
||||
else:
|
||||
status = "EMPTY"
|
||||
print(f" cache after unbounded: {status}")
|
||||
|
||||
# ── chunk-level streaming + multi-stage pipe backpressure ─────────
|
||||
print("\n=== STREAMING (single command) ===")
|
||||
target = "/gcs/data/example.jsonl"
|
||||
r = await ws.execute(f"stat -c '%s' {target}")
|
||||
size = int((await r.stdout_str()).strip())
|
||||
print(f" object size: {size:,} bytes")
|
||||
|
||||
async def measure(label: str, cmd: str) -> None:
|
||||
before = sum(rec.bytes for rec in ws.ops.records)
|
||||
t0 = time.monotonic()
|
||||
r = await ws.execute(cmd)
|
||||
dt = time.monotonic() - t0
|
||||
net = sum(rec.bytes for rec in ws.ops.records) - before
|
||||
head = (await r.stdout_str()).strip().splitlines()
|
||||
first = head[0][:48] if head else ""
|
||||
print(f" {label:42s} bytes={net:>10,} t={dt:4.2f}s "
|
||||
f"lines={len(head):>4} out0={json.dumps(first)}")
|
||||
|
||||
await ws.cache.clear()
|
||||
await measure("head -n 1 (line-streamed)", f"head -n 1 {target}")
|
||||
await ws.cache.clear()
|
||||
await measure("head -c 100 (byte-range)", f"head -c 100 {target}")
|
||||
await ws.cache.clear()
|
||||
await measure("grep -m 1 (early-exit)", f"grep -m 1 mirage {target}")
|
||||
|
||||
print("\n=== STREAMING CHAIN (multi-stage pipe backpressure) ===")
|
||||
await ws.cache.clear()
|
||||
await measure("cat | head -n 1", f"cat {target} | head -n 1")
|
||||
await ws.cache.clear()
|
||||
await measure("cat | tr A-Z a-z | head -n 1",
|
||||
f"cat {target} | tr A-Z a-z | head -n 1")
|
||||
await ws.cache.clear()
|
||||
await measure("cat | grep mirage | head -n 1",
|
||||
f"cat {target} | grep mirage | head -n 1")
|
||||
await ws.cache.clear()
|
||||
await measure("4-stage: cat|tr|grep|head -n 1",
|
||||
f"cat {target} | tr A-Z a-z | grep mirage | head -n 1")
|
||||
await ws.cache.clear()
|
||||
await measure(
|
||||
"5-stage: cat|tr|grep|head|wc -l",
|
||||
f"cat {target} | tr A-Z a-z | grep mirage | head -n 1 "
|
||||
"| wc -l")
|
||||
await ws.cache.clear()
|
||||
await measure("non-cancellable: cat | wc -l", f"cat {target} | wc -l")
|
||||
|
||||
print(f"\nStats: {ops_summary()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from mirage import Mount, MountMode, Workspace
|
||||
from mirage.resource.gcs import GCSConfig, GCSResource
|
||||
|
||||
load_dotenv(".env.development")
|
||||
|
||||
config = GCSConfig(
|
||||
bucket=os.environ["GCS_BUCKET"],
|
||||
access_key_id=os.environ["GCS_ACCESS_KEY_ID"],
|
||||
secret_access_key=os.environ["GCS_SECRET_ACCESS_KEY"],
|
||||
)
|
||||
|
||||
resource = GCSResource(config)
|
||||
|
||||
with Workspace({"/gcs/": Mount(resource, mode=MountMode.READ,
|
||||
fuse=True)}) as ws:
|
||||
mp = ws.fuse_mountpoint
|
||||
|
||||
print(f"=== FUSE MODE: mounted at {mp} ===\n")
|
||||
|
||||
print("--- os.listdir() ---")
|
||||
entries = os.listdir(f"{mp}/data")
|
||||
for e in entries:
|
||||
print(f" {e}")
|
||||
|
||||
print("\n--- open() + read example.json (first 5 lines) ---")
|
||||
with open(f"{mp}/data/example.json") as f:
|
||||
for i, line in enumerate(f):
|
||||
if i >= 5:
|
||||
break
|
||||
print(f" {line.rstrip()[:120]}")
|
||||
|
||||
print("\n--- open() + read example.jsonl (first 3 lines) ---")
|
||||
with open(f"{mp}/data/example.jsonl") as f:
|
||||
for i, line in enumerate(f):
|
||||
if i >= 3:
|
||||
break
|
||||
rec = json.loads(line)
|
||||
print(f" [{i}] {json.dumps(rec)[:100]}...")
|
||||
|
||||
print("\n--- os.path.getsize() ---")
|
||||
size = os.path.getsize(f"{mp}/data/example.json")
|
||||
print(f" example.json: {size} bytes")
|
||||
|
||||
print(f"\n>>> FUSE mounted at: {mp}")
|
||||
print(">>> Open another terminal and run:")
|
||||
print(f">>> ls {mp}/data/")
|
||||
print(f">>> cat {mp}/data/example.json")
|
||||
print(">>> Press Enter to unmount and exit...")
|
||||
input()
|
||||
|
||||
records = ws.ops.records
|
||||
total = sum(r.bytes for r in records)
|
||||
print(f"\nStats: {len(records)} ops, {total} bytes")
|
||||
@@ -0,0 +1,89 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from mirage import MountMode, Workspace
|
||||
from mirage.resource.gcs import GCSConfig, GCSResource
|
||||
|
||||
load_dotenv(".env.development")
|
||||
|
||||
config = GCSConfig(
|
||||
bucket=os.environ["GCS_BUCKET"],
|
||||
access_key_id=os.environ["GCS_ACCESS_KEY_ID"],
|
||||
secret_access_key=os.environ["GCS_SECRET_ACCESS_KEY"],
|
||||
)
|
||||
|
||||
resource = GCSResource(config)
|
||||
|
||||
|
||||
async def main():
|
||||
with Workspace({"/gcs/": resource}, mode=MountMode.READ) as ws:
|
||||
vos = sys.modules["os"]
|
||||
print("=== VFS MODE: open() reads from GCS transparently ===\n")
|
||||
|
||||
print("--- os.listdir() root ---")
|
||||
root = vos.listdir("/gcs")
|
||||
for e in root:
|
||||
print(f" {e}")
|
||||
|
||||
print("\n--- os.path.isdir() on prefix ---")
|
||||
print(f" /gcs/data: {vos.path.isdir('/gcs/data')}")
|
||||
|
||||
print("\n--- os.listdir() data ---")
|
||||
entries = vos.listdir("/gcs/data")
|
||||
for e in entries:
|
||||
print(f" {e}")
|
||||
|
||||
print("\n--- open() + read example.json (first 5 lines) ---")
|
||||
with open("/gcs/data/example.json") as f:
|
||||
for i, line in enumerate(f):
|
||||
if i >= 5:
|
||||
break
|
||||
print(f" {line.rstrip()[:120]}")
|
||||
|
||||
print("\n--- open() + read example.jsonl (first 3 lines) ---")
|
||||
with open("/gcs/data/example.jsonl") as f:
|
||||
for i, line in enumerate(f):
|
||||
if i >= 3:
|
||||
break
|
||||
rec = json.loads(line)
|
||||
print(f" [{i}] {json.dumps(rec)[:100]}...")
|
||||
|
||||
print("\n--- os.path.exists() ---")
|
||||
print(f" example.json: {vos.path.exists('/gcs/data/example.json')}")
|
||||
print(f" nonexistent: {vos.path.exists('/gcs/data/nope.txt')}")
|
||||
|
||||
print("\n--- VFS commands ---")
|
||||
result = await ws.execute("grep -c mirage /gcs/data/example.jsonl")
|
||||
print(f" grep matches: {(await result.stdout_str()).strip()}")
|
||||
|
||||
print("\n--- bash history ---")
|
||||
with open("/.bash_history") as f:
|
||||
for i, line in enumerate(f):
|
||||
if i >= 6:
|
||||
break
|
||||
print(f" {line.rstrip()[:120]}")
|
||||
|
||||
records = ws.ops.records
|
||||
total = sum(r.bytes for r in records)
|
||||
print(f"\nStats: {len(records)} ops, {total} bytes transferred")
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user