chore: import upstream snapshot with attribution

doc/source/serve/doc_code/http_guide/disconnects.py

@@ -0,0 +1,121 @@
+# flake8: noqa
+# fmt: off
+
+import ray
+import sys
+from typing import List
+
+from ray._common.test_utils import wait_for_condition
+
+
+# Overwrite print statement to make doc code testable
+@ray.remote
+class PrintStorage:
+    def __init__(self):
+        self.print_storage: List[str] = []
+
+    def add(self, s: str):
+        self.print_storage.append(s)
+
+    def clear(self):
+        self.print_storage.clear()
+
+    def get(self) -> List[str]:
+        return self.print_storage
+
+
+print_storage_handle = PrintStorage.remote()
+
+
+def print(string: str):
+    ray.get(print_storage_handle.add.remote(string))
+    sys.stdout.write(f"{string}\n")
+
+
+# __start_basic_disconnect__
+import asyncio
+from ray import serve
+
+
+@serve.deployment
+async def startled():
+    try:
+        print("Replica received request!")
+        await asyncio.sleep(10000)
+    except asyncio.CancelledError:
+        # Add custom behavior that should run
+        # upon cancellation here.
+        print("Request got cancelled!")
+# __end_basic_disconnect__
+
+serve.run(startled.bind())
+
+import requests
+from requests.exceptions import Timeout
+
+# Intentionally time out request to test cancellation behavior
+try:
+    requests.get("http://localhost:8000", timeout=0.5)
+except Timeout:
+    pass
+
+wait_for_condition(
+    lambda: {"Replica received request!", "Request got cancelled!"}
+    == set(ray.get(print_storage_handle.get.remote())),
+    timeout=5,
+)
+
+sys.stdout.write(f"{ray.get(print_storage_handle.get.remote())}\n")
+
+ray.get(print_storage_handle.clear.remote())
+
+
+# __start_shielded_disconnect__
+import asyncio
+from ray import serve
+
+
+@serve.deployment
+class SnoringSleeper:
+    async def snore(self):
+        await asyncio.sleep(1)
+        print("ZZZ")
+
+    async def __call__(self):
+        try:
+            print("SnoringSleeper received request!")
+
+            # Prevent the snore() method from being cancelled
+            await asyncio.shield(self.snore())
+
+        except asyncio.CancelledError:
+            print("SnoringSleeper's request was cancelled!")
+
+
+app = SnoringSleeper.bind()
+# __end_shielded_disconnect__
+
+serve.run(app)
+
+import requests
+from requests.exceptions import Timeout
+
+# Intentionally time out request to test cancellation behavior
+try:
+    requests.get("http://localhost:8000", timeout=0.5)
+except Timeout:
+    pass
+
+wait_for_condition(
+    lambda: {
+        "SnoringSleeper received request!",
+        "SnoringSleeper's request was cancelled!",
+        "ZZZ",
+    }
+    == set(ray.get(print_storage_handle.get.remote())),
+    timeout=5,
+)
+
+sys.stdout.write(f"{ray.get(print_storage_handle.get.remote())}\n")
+
+ray.get(print_storage_handle.clear.remote())

doc/source/serve/doc_code/http_guide/http_guide.py

@@ -0,0 +1,176 @@
+# flake8: noqa
+
+# __begin_starlette__
+import starlette.requests
+import requests
+from ray import serve
+
+
+@serve.deployment
+class Counter:
+    def __call__(self, request: starlette.requests.Request):
+        return request.query_params
+
+
+serve.run(Counter.bind())
+resp = requests.get("http://localhost:8000?a=b&c=d")
+assert resp.json() == {"a": "b", "c": "d"}
+# __end_starlette__
+
+# __begin_fastapi__
+import ray
+import requests
+from fastapi import FastAPI
+from ray import serve
+
+app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(app)
+class MyFastAPIDeployment:
+    @app.get("/")
+    def root(self):
+        return "Hello, world!"
+
+
+serve.run(MyFastAPIDeployment.bind(), route_prefix="/hello")
+resp = requests.get("http://localhost:8000/hello")
+assert resp.json() == "Hello, world!"
+# __end_fastapi__
+
+
+# __begin_fastapi_multi_routes__
+import ray
+import requests
+from fastapi import FastAPI
+from ray import serve
+
+app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(app)
+class MyFastAPIDeployment:
+    @app.get("/")
+    def root(self):
+        return "Hello, world!"
+
+    @app.post("/{subpath}")
+    def root(self, subpath: str):
+        return f"Hello from {subpath}!"
+
+
+serve.run(MyFastAPIDeployment.bind(), route_prefix="/hello")
+resp = requests.post("http://localhost:8000/hello/Serve")
+assert resp.json() == "Hello from Serve!"
+# __end_fastapi_multi_routes__
+
+# __begin_byo_fastapi__
+import ray
+import requests
+from fastapi import FastAPI
+from ray import serve
+
+app = FastAPI()
+
+
+@app.get("/")
+def f():
+    return "Hello from the root!"
+
+
+@serve.deployment
+@serve.ingress(app)
+class FastAPIWrapper:
+    pass
+
+
+serve.run(FastAPIWrapper.bind(), route_prefix="/")
+resp = requests.get("http://localhost:8000/")
+assert resp.json() == "Hello from the root!"
+# __end_byo_fastapi__
+
+# __begin_fastapi_middleware__
+import requests
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from ray import serve
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["https://example.com"],
+    allow_methods=["GET", "POST"],
+)
+
+
+@serve.deployment
+@serve.ingress(app)
+class Ingress:
+    @app.get("/")
+    def root(self):
+        return "ok"
+
+
+serve.run(Ingress.bind())
+resp = requests.get(
+    "http://localhost:8000/",
+    headers={"Origin": "https://example.com"},
+)
+assert resp.json() == "ok"
+assert resp.headers["access-control-allow-origin"] == "https://example.com"
+# __end_fastapi_middleware__
+
+
+# __begin_fastapi_factory_pattern__
+import requests
+from fastapi import FastAPI
+from ray import serve
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+
+
+@serve.deployment
+class ChildDeployment:
+    def __call__(self):
+        return "Hello from the child deployment!"
+
+
+def fastapi_factory():
+    """Factory-style FastAPI app used as Serve ingress.
+
+    We build the FastAPI app inside a factory and pass the callable to
+    @serve.ingress.
+    """
+    app = FastAPI()
+
+    # In an object-based ingress (where the FastAPI app is stored on the
+    # deployment instance), Ray would need to serialize the app and its
+    # instrumentation. Some instrumentors (like FastAPIInstrumentor) are not
+    # picklable, which can cause serialization failures. Creating and
+    # instrumenting the app here sidesteps that issue.
+    FastAPIInstrumentor.instrument_app(app)
+
+    @app.get("/")
+    async def root():
+        # Handlers defined inside this factory don't have access to the
+        # ParentDeployment instance (i.e., there's no `self` here), so we
+        # can't call `self.child`. Instead, fetch a handle by deployment name.
+        handle = serve.get_deployment_handle("ChildDeployment", app_name="default")
+        return {"message": await handle.remote()}
+
+    return app
+
+
+@serve.deployment
+@serve.ingress(fastapi_factory)
+class ParentDeployment:
+    def __init__(self, child):
+        self.child = child
+
+
+serve.run(ParentDeployment.bind(ChildDeployment.bind()))
+
+resp = requests.get("http://localhost:8000/")
+assert resp.json() == {"message": "Hello from the child deployment!"}
+# __end_fastapi_factory_pattern__

doc/source/serve/doc_code/http_guide/streaming_example.py

@@ -0,0 +1,82 @@
+# flake8: noqa
+
+# __begin_example__
+import time
+from typing import Generator
+
+import requests
+from starlette.responses import StreamingResponse
+from starlette.requests import Request
+
+from ray import serve
+
+
+@serve.deployment
+class StreamingResponder:
+    def generate_numbers(self, max: int) -> Generator[str, None, None]:
+        for i in range(max):
+            yield str(i)
+            time.sleep(0.1)
+
+    def __call__(self, request: Request) -> StreamingResponse:
+        max = request.query_params.get("max", "25")
+        gen = self.generate_numbers(int(max))
+        return StreamingResponse(gen, status_code=200, media_type="text/plain")
+
+
+serve.run(StreamingResponder.bind())
+
+r = requests.get("http://localhost:8000?max=10", stream=True)
+start = time.time()
+r.raise_for_status()
+for chunk in r.iter_content(chunk_size=None, decode_unicode=True):
+    print(f"Got result {round(time.time()-start, 1)}s after start: '{chunk}'")
+# __end_example__
+
+
+r = requests.get("http://localhost:8000?max=10", stream=True)
+r.raise_for_status()
+for i, chunk in enumerate(r.iter_content(chunk_size=None, decode_unicode=True)):
+    assert chunk == str(i)
+
+
+# __begin_cancellation__
+import asyncio
+import time
+from typing import AsyncGenerator
+
+import requests
+from starlette.responses import StreamingResponse
+from starlette.requests import Request
+
+from ray import serve
+
+
+@serve.deployment
+class StreamingResponder:
+    async def generate_forever(self) -> AsyncGenerator[str, None]:
+        try:
+            i = 0
+            while True:
+                yield str(i)
+                i += 1
+                await asyncio.sleep(0.1)
+        except asyncio.CancelledError:
+            print("Cancelled! Exiting.")
+
+    def __call__(self, request: Request) -> StreamingResponse:
+        gen = self.generate_forever()
+        return StreamingResponse(gen, status_code=200, media_type="text/plain")
+
+
+serve.run(StreamingResponder.bind())
+
+r = requests.get("http://localhost:8000?max=10", stream=True)
+start = time.time()
+r.raise_for_status()
+for i, chunk in enumerate(r.iter_content(chunk_size=None, decode_unicode=True)):
+    print(f"Got result {round(time.time()-start, 1)}s after start: '{chunk}'")
+    if i == 10:
+        print("Client disconnecting")
+        break
+# __end_cancellation__

doc/source/serve/doc_code/http_guide/websockets_example.py

@@ -0,0 +1,39 @@
+# flake8: noqa
+
+# __websocket_serve_app_start__
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+
+from ray import serve
+
+
+app = FastAPI()
+
+
+@serve.deployment
+@serve.ingress(app)
+class EchoServer:
+    @app.websocket("/")
+    async def echo(self, ws: WebSocket):
+        await ws.accept()
+
+        try:
+            while True:
+                text = await ws.receive_text()
+                await ws.send_text(text)
+        except WebSocketDisconnect:
+            print("Client disconnected.")
+
+
+serve_app = serve.run(EchoServer.bind())
+# __websocket_serve_app_end__
+
+# __websocket_serve_client_start__
+from websockets.sync.client import connect
+
+with connect("ws://localhost:8000") as websocket:
+    websocket.send("Eureka!")
+    assert websocket.recv() == "Eureka!"
+
+    websocket.send("I've found it!")
+    assert websocket.recv() == "I've found it!"
+# __websocket_serve_client_end__