strukto-ai--mirage/examples/python/s3/s3_data.py

# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

import asyncio
import os

from dotenv import load_dotenv

from mirage import MountMode, Workspace
from mirage.resource.s3 import S3Config, S3Resource

load_dotenv(".env.development")

config = S3Config(
    bucket=os.environ["AWS_S3_BUCKET"],
    region=os.environ.get("AWS_DEFAULT_REGION", "us-east-1"),
    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
)

backend = S3Resource(config)
ws = Workspace({"/s3/": backend}, mode=MountMode.READ)


def ops_summary() -> str:
    records = ws.ops.records
    total = sum(r.bytes for r in records)
    return f"{len(records)} ops, {total} bytes transferred"


PARQUET = "/s3/data/example.parquet"
ORC = "/s3/data/example.orc"
FEATHER = "/s3/data/example.feather"
HDF5 = "/s3/data/example.h5"


async def main():
    print("=== PLAN: estimate bytes before execution ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        print(f"--- {label} ---")
        for cmd in [
                f"cat {path}", f"head -n 3 {path}", f"wc -l {path}",
                f"stat {path}"
        ]:
            dr = await ws.execute(cmd, provision=True)
            net = dr.network_read
            cache = dr.cache_read
            print(f"  {cmd.split()[0]:6s}: net={net}, cache={cache}, "
                  f"ops={dr.read_ops}, {dr.precision}")
        print()

    print(f"Stats after plans (should be 0): {ops_summary()}\n")

    print("=== CAT: schema + preview ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        print(f"--- cat {label} ---")
        result = await ws.execute(f"cat {path}")
        print(await result.stdout_str())

    print(f"Stats after cat: {ops_summary()}\n")

    print("=== HEAD: first 3 rows ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        print(f"--- head -n 3 {label} ---")
        result = await ws.execute(f"head -n 3 {path}")
        print(await result.stdout_str())

    print("=== TAIL: last 3 rows ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        print(f"--- tail -n 3 {label} ---")
        result = await ws.execute(f"tail -n 3 {path}")
        print(await result.stdout_str())

    print("=== WC: row count ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        result = await ws.execute(f"wc -l {path}")
        print(f"  {label}: {(await result.stdout_str()).strip()} rows")
    print()

    print("=== STAT: metadata ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        print(f"--- stat {label} ---")
        result = await ws.execute(f"stat {path}")
        print(await result.stdout_str())

    print("=== GREP: search for 'item_5' ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        print(f"--- grep item_5 {label} ---")
        result = await ws.execute(f"grep item_5 {path}")
        print(f"  {(await result.stdout_str()).strip()}")
    print()

    print("=== CUT: select 'id,label' columns ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        print(f"--- cut -f id,label {label} ---")
        result = await ws.execute(f"cut -f id,label {path}")
        print(await result.stdout_str())

    print("=== CACHE: plan after data is cached ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        dr = await ws.execute(f"cat {path}", provision=True)
        print(f"  {label}: net={dr.network_read}, cache={dr.cache_read}, "
              f"hits={dr.cache_hits}")
    print()

    print("=== PIPES: filetype commands in pipelines ===\n")

    print("--- cat parquet | grep item_3 ---")
    result = await ws.execute(f"cat {PARQUET} | grep item_3")
    print(f"  {(await result.stdout_str()).strip()}")

    print("\n--- head -n 5 parquet | wc -l ---")
    result = await ws.execute(f"head -n 5 {PARQUET} | wc -l")
    print(f"  {(await result.stdout_str()).strip()}")

    print("\n=== LS -L: long listing ===\n")

    result = await ws.execute("ls -l /s3/data/")
    print(await result.stdout_str())

    print("=== FILE: rich type detection ===\n")

    for label, path in [("parquet", PARQUET), ("orc", ORC),
                        ("feather", FEATHER), ("hdf5", HDF5)]:
        result = await ws.execute(f"file {path}")
        print(f"  {label}: {(await result.stdout_str()).strip()}")
    print()

    print("=== GLOB: grep / head across files via *.ext ===\n")

    print("--- grep item_5 /s3/data/*.parquet ---")
    result = await ws.execute("grep item_5 /s3/data/*.parquet")
    print(f"  exit={result.exit_code}")
    for line in (await result.stdout_str()).strip().splitlines():
        print(f"  {line}")

    print("\n--- grep item_3 /s3/data/*.orc ---")
    result = await ws.execute("grep item_3 /s3/data/*.orc")
    print(f"  exit={result.exit_code}")
    for line in (await result.stdout_str()).strip().splitlines():
        print(f"  {line}")

    print("\n--- head -n 2 /s3/data/*.feather ---")
    result = await ws.execute("head -n 2 /s3/data/*.feather")
    print(f"  exit={result.exit_code}")
    for line in (await result.stdout_str()).strip().splitlines():
        print(f"  {line}")

    print(f"\nFinal stats: {ops_summary()}")


asyncio.run(main())