chore: import upstream snapshot with attribution

2026-07-13 12:47:58 +08:00
commit b16403ea71
789 changed files with 115226 additions and 0 deletions
@@ -0,0 +1,57 @@
+import asyncio
+import zlib
+from typing import IO, AsyncIterable, AsyncIterator, Iterable, Iterator
+
+IO_CHUNK_SIZE = 65_536
+
+
+def iter_io_chunks(data: IO) -> Iterator[bytes]:
+    """Read a file-like object in chunks, encoding text chunks to UTF-8."""
+    while True:
+        chunk = data.read(IO_CHUNK_SIZE)
+        if not chunk:
+            break
+        yield chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
+
+
+async def aiter_io_chunks(data: IO) -> AsyncIterator[bytes]:
+    """Read a file-like object in chunks, encoding text chunks to UTF-8.
+
+    `data.read` is a synchronous (potentially disk-blocking) call, so it runs in
+    a worker thread to avoid stalling the event loop during large uploads.
+    """
+    while True:
+        chunk = await asyncio.to_thread(data.read, IO_CHUNK_SIZE)
+        if not chunk:
+            break
+        yield chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
+
+
+def _gzip_compressor():
+    # wbits > 16 makes zlib produce a gzip-formatted stream.
+    return zlib.compressobj(wbits=zlib.MAX_WBITS | 16)
+
+
+def gzip_iter(chunks: Iterable[bytes]) -> Iterator[bytes]:
+    """Gzip-compress a byte stream chunk by chunk."""
+    compressor = _gzip_compressor()
+    for chunk in chunks:
+        compressed = compressor.compress(chunk)
+        if compressed:
+            yield compressed
+    yield compressor.flush()
+
+
+async def agzip_iter(chunks: AsyncIterable[bytes]) -> AsyncIterator[bytes]:
+    """Gzip-compress a byte stream chunk by chunk.
+
+    Compression is CPU-bound, so it runs in a worker thread to avoid stalling
+    the event loop during large uploads (zlib releases the GIL while
+    compressing, so the offload genuinely overlaps with the loop).
+    """
+    compressor = _gzip_compressor()
+    async for chunk in chunks:
+        compressed = await asyncio.to_thread(compressor.compress, chunk)
+        if compressed:
+            yield compressed
+    yield await asyncio.to_thread(compressor.flush)