chore: import upstream snapshot with attribution

studio/backend/plugins/data-designer-github-repo-seed/README.md

@@ -0,0 +1,73 @@
+# data-designer-github-repo-seed
+
+A Data Designer seed-reader plugin for **Unsloth Studio** that scrapes real
+GitHub data (issues, pull requests, commits) from one or more repositories
+and hands it to the recipe pipeline as a seed dataset.
+
+Designed to ship with Studio as a default seed source so any user with a
+GitHub token can build training datasets straight from live repos.
+
+## What it does
+
+Given a list of `owner/name` repos, a GitHub token, and a per-resource
+`limit`, the plugin uses GitHub's GraphQL API to fetch issues, pull
+requests, and/or commits, with labels, state, authors, and the first N
+comments of each item, and materialises a single JSONL with uniform
+columns so the rest of the recipe (LLM text / LLM structured / processors)
+can treat it like any other seed table.
+
+| Column        | Description                                    |
+|---------------|------------------------------------------------|
+| `item_type`   | `issue` / `pull` / `commit`                    |
+| `repo`        | `owner/name`                                   |
+| `number`      | Issue/PR number, or commit SHA                 |
+| `title`       | Title (or commit message headline)             |
+| `body`        | Issue/PR body (or full commit message)         |
+| `state`       | `OPEN` / `CLOSED` / `MERGED` (empty for commit)|
+| `author`      | GitHub login of the author                     |
+| `created_at`  | ISO8601                                        |
+| `closed_at`   | ISO8601 (empty for commits)                    |
+| `url`         | Permalink                                      |
+| `labels`      | List of label names                            |
+| `comments`    | First N comments concatenated                  |
+
+## Usage in a recipe
+
+```json
+{
+  "seed_config": {
+    "source": {
+      "seed_type": "github_repo",
+      "repos": ["unslothai/unsloth", "unslothai/unsloth-zoo"],
+      "token": "",
+      "item_types": ["issues", "pulls"],
+      "limit": 100,
+      "include_comments": true,
+      "max_comments_per_item": 30
+    },
+    "sampling_strategy": "shuffle",
+    "selection_strategy": null
+  }
+}
+```
+
+Leave `token` empty to fall back to the server's `GH_TOKEN` / `GITHUB_TOKEN`
+environment variable, useful when the recipe is published and shouldn't
+carry a secret.
+
+## Auth
+
+A GitHub personal access token with `public_repo` scope is enough for public
+repositories; `repo` scope is required for private ones. GraphQL requests
+are rate-limit aware: the client inspects `x-ratelimit-*` headers and
+sleeps until reset when the budget drops below a safety threshold.
+
+## Install
+
+Shipped as a default Studio plugin. For development:
+
+```bash
+pip install -e .
+```
+
+Registered automatically via the `data_designer.plugins` entry point.

studio/backend/plugins/data-designer-github-repo-seed/pyproject.toml

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "data-designer-github-repo-seed"
+version = "0.1.0"
+description = "Unsloth Studio seed plugin that scrapes GitHub issues, PRs, and commits."
+requires-python = ">=3.11"
+dependencies = [
+  "data-designer-engine>=0.5.4,<0.6",
+  "requests>=2.31",
+]
+
+[project.entry-points."data_designer.plugins"]
+github_repo_seed = "data_designer_github_repo_seed.plugin:github_repo_seed_plugin"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+# Intentionally empty. Data-designer loads submodules lazily via qualified names
+# in plugin.py, so importing this package must not touch data_designer.engine.*
+# during Studio bootstrap (circular import).

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/config.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import Field, field_validator, model_validator
+
+from data_designer.config.seed_source import SeedSource
+
+
+class GitHubRepoSeedSource(SeedSource):
+    seed_type: Literal["github_repo"] = "github_repo"
+
+    repos: list[str] = Field(
+        default_factory = list,
+        description = "List of GitHub repositories to scrape, each in `owner/name` form.",
+    )
+    token: str = Field(
+        default = "",
+        description = "Personal access token. Leave blank to read GH_TOKEN / GITHUB_TOKEN from env at run time.",
+    )
+    item_types: list[Literal["issues", "pulls", "commits"]] = Field(
+        default = ["issues", "pulls"],
+        description = "Which GitHub item types to fetch per repo.",
+    )
+    limit: int = Field(
+        default = 100,
+        ge = 1,
+        le = 5000,
+        description = "Maximum items per repo per item type (e.g. limit=100 + ['issues','pulls'] => up to 200 items per repo).",
+    )
+    include_comments: bool = Field(
+        default = True,
+        description = "Fetch the first N comments of each issue/PR and include them in the `comments` column.",
+    )
+    max_comments_per_item: int = Field(default = 30, ge = 0, le = 200)
+
+    @field_validator("repos")
+    @classmethod
+    def _validate_repos(cls, v: list[str]) -> list[str]:
+        out: list[str] = []
+        for r in v or []:
+            r = r.strip()
+            if not r:
+                continue
+            if r.count("/") != 1 or not all(r.split("/")):
+                raise ValueError(f"Each repo must be `owner/name`; got {r!r}")
+            out.append(r)
+        return out
+
+    @field_validator("item_types")
+    @classmethod
+    def _validate_item_types(cls, v: list[str]) -> list[str]:
+        if not v:
+            raise ValueError("item_types must not be empty")
+        return list(dict.fromkeys(v))
+
+    @model_validator(mode = "after")
+    def _ensure_repos(self) -> "GitHubRepoSeedSource":
+        if not self.repos:
+            raise ValueError("At least one repo is required")
+        return self

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py

@@ -0,0 +1,78 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import hashlib
+import tempfile
+import threading
+from pathlib import Path
+from typing import Optional
+
+import data_designer.lazy_heavy_imports as lazy
+from data_designer.engine.resources.seed_reader import SeedReader
+
+from .config import GitHubRepoSeedSource
+from .scraper import ScrapeConfig, materialize_to_jsonl
+
+
+# In-process cache: config signature -> JSONL path. A recipe job reads the seed
+# multiple times (validation, preview, sampling); memoize to avoid re-scraping
+# GitHub on each call. Key uses a short SHA-256 of the token (never the raw value)
+# so token never hits memory twice and rotation invalidates cleanly.
+_SCRAPE_CACHE: dict[tuple, str] = {}
+_SCRAPE_CACHE_LOCK = threading.Lock()
+
+
+def _scrape_cache_key(cfg: ScrapeConfig) -> tuple:
+    token_digest = hashlib.sha256(
+        (cfg.token or "").encode("utf-8"),
+    ).hexdigest()[:16]
+    return (
+        tuple(cfg.repos),
+        tuple(cfg.item_types),
+        cfg.limit,
+        bool(cfg.include_comments),
+        cfg.max_comments_per_item,
+        token_digest,
+    )
+
+
+def _lookup_cached_scrape(key: tuple) -> Optional[str]:
+    with _SCRAPE_CACHE_LOCK:
+        path = _SCRAPE_CACHE.get(key)
+    if path and Path(path).exists():
+        return path
+    # Stale entry (tmp cleanup/restart); drop it so the caller re-materializes.
+    if path:
+        with _SCRAPE_CACHE_LOCK:
+            _SCRAPE_CACHE.pop(key, None)
+    return None
+
+
+def _store_cached_scrape(key: tuple, path: str) -> None:
+    with _SCRAPE_CACHE_LOCK:
+        _SCRAPE_CACHE[key] = path
+
+
+class GitHubRepoSeedReader(SeedReader[GitHubRepoSeedSource]):
+    def create_duckdb_connection(self):
+        return lazy.duckdb.connect()
+
+    def get_dataset_uri(self) -> str:
+        out_dir = Path(tempfile.gettempdir()) / "studio-github-repo-seed"
+        cfg = ScrapeConfig(
+            repos = list(self.source.repos),
+            token = self.source.token,
+            item_types = list(self.source.item_types),
+            limit = self.source.limit,
+            include_comments = self.source.include_comments,
+            max_comments_per_item = self.source.max_comments_per_item,
+        )
+        cache_key = _scrape_cache_key(cfg)
+        cached_path = _lookup_cached_scrape(cache_key)
+        if cached_path is not None:
+            return cached_path
+        path = materialize_to_jsonl(cfg, out_dir)
+        _store_cached_scrape(cache_key, str(path))
+        return str(path)

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/plugin.py

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from data_designer.plugins.plugin import Plugin, PluginType
+
+github_repo_seed_plugin = Plugin(
+    impl_qualified_name = "data_designer_github_repo_seed.impl.GitHubRepoSeedReader",
+    config_qualified_name = "data_designer_github_repo_seed.config.GitHubRepoSeedSource",
+    plugin_type = PluginType.SEED_READER,
+)

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper.py

@@ -0,0 +1,233 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Multi-repo GitHub scraper for the Studio seed plugin.
+
+Drives the GraphQL scraper in `scraper_impl/` per repo, capped via trial_limits
+to stop at `limit` items per resource. Then reads the per-resource JSONL shards
+and flattens them into one unified JSONL with stable columns (`item_type`,
+`repo`, `number`, `title`, `body`, ...).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+
+# Defer scraper_impl imports until scrape() has a resolved token.
+_IMPL_DIR = Path(__file__).parent / "scraper_impl"
+
+
+def _ensure_impl_on_path() -> None:
+    if str(_IMPL_DIR) not in sys.path:
+        sys.path.insert(0, str(_IMPL_DIR))
+
+
+def _load_impl():
+    _ensure_impl_on_path()
+    import importlib
+
+    gh_client = importlib.import_module("gh_client")  # type: ignore
+    scraper_mod = importlib.import_module("scraper")  # type: ignore
+    return gh_client.GitHubClient, scraper_mod.RepoScraper
+
+
+@dataclass
+class ScrapeConfig:
+    repos: list[str]
+    token: str
+    item_types: list[str]
+    limit: int
+    include_comments: bool
+    max_comments_per_item: int
+
+
+@dataclass(frozen = True)
+class ResolvedToken:
+    value: str
+    source: str
+
+
+def _resolve_token(token: str) -> ResolvedToken:
+    if token:
+        return ResolvedToken(
+            value = token,
+            source = "explicit token argument (recipe-level field)",
+        )
+    if os.environ.get("GH_TOKEN"):
+        return ResolvedToken(
+            value = os.environ["GH_TOKEN"],
+            source = "GH_TOKEN environment variable",
+        )
+    if os.environ.get("GITHUB_TOKEN"):
+        return ResolvedToken(
+            value = os.environ["GITHUB_TOKEN"],
+            source = "GITHUB_TOKEN environment variable",
+        )
+    raise ValueError(
+        "GitHub token is required. Set it in the recipe config or the GH_TOKEN / GITHUB_TOKEN env var."
+    )
+
+
+def _read_jsonl(path: Path, max_rows: int | None = None):
+    if not path.exists():
+        return
+    with path.open(encoding = "utf-8") as f:
+        for i, line in enumerate(f):
+            if not line.strip():
+                continue
+            if max_rows is not None and i >= max_rows:
+                return
+            try:
+                yield json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+
+def _flatten_issue_row(r: dict, repo: str, include_comments: bool, max_c: int) -> dict:
+    labels = [l.get("name") for l in (r.get("labels", {}) or {}).get("nodes", []) if l.get("name")]
+    comments_nodes = (r.get("comments") or {}).get("nodes") or []
+    comments_text = ""
+    if include_comments and comments_nodes:
+        kept = comments_nodes[:max_c]
+        comments_text = "\n\n".join(
+            f"[{(c.get('author') or {}).get('login', '?')}]: {c.get('body') or ''}" for c in kept
+        )
+    return {
+        "item_type": "issue",
+        "repo": repo,
+        "number": r.get("number"),
+        "title": r.get("title") or "",
+        "body": r.get("body") or "",
+        "state": r.get("state") or "",
+        "author": (r.get("author") or {}).get("login", ""),
+        "created_at": r.get("createdAt") or "",
+        "closed_at": r.get("closedAt") or "",
+        "url": r.get("url") or r.get("permalink") or "",
+        "labels": labels,
+        "comments": comments_text,
+    }
+
+
+def _flatten_pr_row(r: dict, repo: str, include_comments: bool, max_c: int) -> dict:
+    labels = [l.get("name") for l in (r.get("labels", {}) or {}).get("nodes", []) if l.get("name")]
+    comments_nodes = (r.get("comments") or {}).get("nodes") or []
+    comments_text = ""
+    if include_comments and comments_nodes:
+        kept = comments_nodes[:max_c]
+        comments_text = "\n\n".join(
+            f"[{(c.get('author') or {}).get('login', '?')}]: {c.get('body') or ''}" for c in kept
+        )
+    return {
+        "item_type": "pull",
+        "repo": repo,
+        "number": r.get("number"),
+        "title": r.get("title") or "",
+        "body": r.get("body") or "",
+        "state": r.get("state") or "",
+        "author": (r.get("author") or {}).get("login", ""),
+        "created_at": r.get("createdAt") or "",
+        "closed_at": r.get("closedAt") or "",
+        "url": r.get("url") or r.get("permalink") or "",
+        "labels": labels,
+        "comments": comments_text,
+    }
+
+
+def _flatten_commit_row(r: dict, repo: str) -> dict:
+    msg = r.get("messageHeadline") or r.get("message") or ""
+    body = r.get("messageBody") or r.get("message") or msg
+    author = r.get("author") or {}
+    return {
+        "item_type": "commit",
+        "repo": repo,
+        "number": r.get("oid") or r.get("sha") or "",
+        "title": msg,
+        "body": body,
+        "state": "",
+        "author": (author.get("user") or {}).get("login") or author.get("name", ""),
+        "created_at": (author.get("date") or r.get("committedDate") or ""),
+        "closed_at": "",
+        "url": r.get("url") or "",
+        "labels": [],
+        "comments": "",
+    }
+
+
+def scrape(cfg: ScrapeConfig, base_dir: Path):
+    token = _resolve_token(cfg.token)
+    GitHubClient, RepoScraper = _load_impl()
+    client = GitHubClient(token = token.value, token_source = token.source)
+    base_dir.mkdir(parents = True, exist_ok = True)
+
+    # Per-resource limits; limit <= 0 means "all" (large cap).
+    effective_limit = cfg.limit if cfg.limit and cfg.limit > 0 else 1_000_000
+    trial_limits: dict[str, int] = {}
+    if "issues" in cfg.item_types:
+        trial_limits["issues"] = effective_limit
+    if "pulls" in cfg.item_types:
+        trial_limits["pull_requests"] = effective_limit
+    if "commits" in cfg.item_types:
+        trial_limits["commits"] = effective_limit
+
+    all_rows: list[dict] = []
+    for repo in cfg.repos:
+        owner, name = repo.split("/", 1)
+        scraper = RepoScraper(
+            owner = owner,
+            name = name,
+            base_dir = base_dir,
+            client = client,
+            trial_limits = trial_limits,
+            light = True,
+        )
+        try:
+            repo_meta = scraper.scrape_repo_meta()
+            if "issues" in cfg.item_types:
+                scraper.scrape_issues()
+            if "pulls" in cfg.item_types:
+                scraper.scrape_prs()
+            if "commits" in cfg.item_types:
+                default_ref = repo_meta.get("defaultBranchRef") or {}
+                default_branch = default_ref.get("name") if isinstance(default_ref, dict) else None
+                branch = f"refs/heads/{default_branch}" if default_branch else "refs/heads/main"
+                scraper.scrape_commits(branch = branch)
+        finally:
+            scraper.close()
+
+        read_cap = cfg.limit if cfg.limit and cfg.limit > 0 else None
+        repo_dir = base_dir / f"{owner}__{name}"
+        if "issues" in cfg.item_types:
+            for row in _read_jsonl(repo_dir / "issues.jsonl", read_cap):
+                all_rows.append(
+                    _flatten_issue_row(row, repo, cfg.include_comments, cfg.max_comments_per_item)
+                )
+        if "pulls" in cfg.item_types:
+            for row in _read_jsonl(repo_dir / "pull_requests.jsonl", read_cap):
+                all_rows.append(
+                    _flatten_pr_row(row, repo, cfg.include_comments, cfg.max_comments_per_item)
+                )
+        if "commits" in cfg.item_types:
+            for row in _read_jsonl(repo_dir / "commits.jsonl", read_cap):
+                all_rows.append(_flatten_commit_row(row, repo))
+
+    return all_rows
+
+
+def materialize_to_jsonl(cfg: ScrapeConfig, out_dir: Path) -> Path:
+    out_dir.mkdir(parents = True, exist_ok = True)
+    tag = "-".join(r.replace("/", "__") for r in cfg.repos)[:120]
+    kinds = "-".join(cfg.item_types)
+    run_id = f"{int(time.time())}-{uuid.uuid4().hex[:12]}"
+    fname = f"github_{tag}__{kinds}__{cfg.limit}_{run_id}.jsonl"
+    out = out_dir / fname
+    rows = scrape(cfg, out_dir / "raw-runs" / run_id)
+    with out.open("w", encoding = "utf-8") as f:
+        for r in rows:
+            f.write(json.dumps(r, ensure_ascii = False) + "\n")
+    return out

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py

@@ -0,0 +1,317 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""GitHub API client with rate-limit awareness, retry, and dual REST/GraphQL support."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import logging
+from datetime import timezone
+from email.utils import parsedate_to_datetime
+from typing import Any, Dict, Iterable, Iterator, List, Optional
+
+import requests
+
+log = logging.getLogger("gh_client")
+
+GRAPHQL_URL = "https://api.github.com/graphql"
+REST_BASE = "https://api.github.com"
+
+BASE_HEADERS = {
+    "Accept": "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28",
+    "User-Agent": "github-data-gatherer/1.0",
+}
+
+
+class RateLimitError(Exception):
+    pass
+
+
+class GitHubAuthError(RuntimeError):
+    """Raised when GitHub returns 401/403 due to invalid or insufficient credentials."""
+
+
+def _retry_after_seconds(value: str | None) -> int | None:
+    if not value:
+        return None
+    try:
+        return max(0, int(value))
+    except ValueError:
+        pass
+    try:
+        retry_at = parsedate_to_datetime(value)
+    except (TypeError, ValueError, IndexError, OverflowError):
+        return None
+    if retry_at.tzinfo is None:
+        retry_at = retry_at.replace(tzinfo = timezone.utc)
+    return max(0, int(retry_at.timestamp() - time.time()))
+
+
+class GitHubClient:
+    def __init__(
+        self,
+        min_remaining_graphql: int = 100,
+        min_remaining_rest: int = 100,
+        token: str | None = None,
+        token_source: str | None = None,
+    ):
+        if token:
+            self._token_source = token_source or "explicit token argument (recipe-level field)"
+        elif os.environ.get("GH_TOKEN"):
+            self._token_source = "GH_TOKEN environment variable"
+            token = os.environ["GH_TOKEN"]
+        elif os.environ.get("GITHUB_TOKEN"):
+            self._token_source = "GITHUB_TOKEN environment variable"
+            token = os.environ["GITHUB_TOKEN"]
+        else:
+            raise RuntimeError("GH_TOKEN or GITHUB_TOKEN not set in environment")
+        self.session = requests.Session()
+        self.session.headers.update({**BASE_HEADERS, "Authorization": f"Bearer {token}"})
+        self.min_remaining_graphql = min_remaining_graphql
+        self.min_remaining_rest = min_remaining_rest
+        self.graphql_remaining: Optional[int] = None
+        self.graphql_reset: Optional[int] = None
+        self.rest_remaining: Optional[int] = None
+        self.rest_reset: Optional[int] = None
+        self.calls_graphql = 0
+        self.calls_rest = 0
+        self.retry_count = 0
+
+    def _sleep_until(
+        self,
+        reset_ts: int,
+        buffer_s: int = 10,
+    ) -> None:
+        now = int(time.time())
+        wait = max(0, reset_ts - now) + buffer_s
+        log.warning("Rate limit hit. Sleeping %ds until reset.", wait)
+        time.sleep(wait)
+
+    def _is_rate_limit_response(self, r: "requests.Response") -> bool:
+        if r.headers.get("Retry-After"):
+            return True
+        if r.headers.get("X-RateLimit-Remaining") == "0":
+            return True
+        body = (r.text or "").lower()
+        return any(
+            marker in body
+            for marker in (
+                "api rate limit exceeded",
+                "rate limit exceeded",
+                "secondary rate limit",
+                "secondary limit",
+                "abuse detection mechanism",
+                "abuse detection",
+            )
+        )
+
+    def _is_auth_failure(self, r: "requests.Response") -> bool:
+        """Tell auth failures apart from rate limiting on 401/403.
+
+        401 is always auth; 403 is auth unless it carries a rate-limit signal
+        (Retry-After, X-RateLimit-Remaining: 0, or abuse/secondary text).
+        """
+        if r.status_code == 401:
+            return True
+        if r.status_code == 403:
+            return not self._is_rate_limit_response(r)
+        return False
+
+    def _raise_auth_error(self, r: "requests.Response", endpoint: str) -> None:
+        snippet = (r.text or "").strip()[:200]
+        request_id = r.headers.get("X-GitHub-Request-Id")
+        request_id_message = f" Request ID: {request_id}." if request_id else ""
+        raise GitHubAuthError(
+            f"GitHub {endpoint} returned {r.status_code} {r.reason}. "
+            f"Token source: {self._token_source}. "
+            f"The token is invalid, expired, or missing required scopes — "
+            f"retrying will not recover.{request_id_message} Response: {snippet}"
+        )
+
+    def _check_rate_and_wait(self, kind: str) -> None:
+        if kind == "graphql":
+            remaining = self.graphql_remaining
+            reset = self.graphql_reset
+            min_remaining = self.min_remaining_graphql
+        else:
+            remaining = self.rest_remaining
+            reset = self.rest_reset
+            min_remaining = self.min_remaining_rest
+        if remaining is not None and remaining < min_remaining:
+            if reset:
+                self._sleep_until(reset)
+                # Reset remaining so we don't spin
+                if kind == "graphql":
+                    self.graphql_remaining = None
+                else:
+                    self.rest_remaining = None
+
+    def graphql(
+        self,
+        query: str,
+        variables: Optional[Dict[str, Any]] = None,
+        max_retries: int = 20,
+    ) -> Dict[str, Any]:
+        self._check_rate_and_wait("graphql")
+        backoff = 2
+        last_err = None
+        for attempt in range(max_retries):
+            try:
+                r = self.session.post(
+                    GRAPHQL_URL,
+                    json = {"query": query, "variables": variables or {}},
+                    timeout = 120,
+                )
+                self.calls_graphql += 1
+                rem = r.headers.get("X-RateLimit-Remaining")
+                rst = r.headers.get("X-RateLimit-Reset")
+                if rem is not None:
+                    try:
+                        self.graphql_remaining = int(rem)
+                    except ValueError:
+                        pass
+                if rst is not None:
+                    try:
+                        self.graphql_reset = int(rst)
+                    except ValueError:
+                        pass
+                if r.status_code in (502, 503, 504):
+                    log.warning("GraphQL %s transient, retrying", r.status_code)
+                    time.sleep(backoff)
+                    backoff = min(backoff * 2, 60)
+                    continue
+                if self._is_auth_failure(r):
+                    self._raise_auth_error(r, "GraphQL")
+                if r.status_code == 403 or r.status_code == 429:
+                    # Secondary/abuse rate limit
+                    retry_after = _retry_after_seconds(r.headers.get("Retry-After"))
+                    if retry_after is not None:
+                        log.warning("Secondary rate limit. Sleep %ds.", retry_after)
+                        time.sleep(retry_after + 2)
+                        continue
+                    if self.graphql_reset:
+                        self._sleep_until(self.graphql_reset)
+                        continue
+                    time.sleep(60)
+                    continue
+                r.raise_for_status()
+                data = r.json()
+                if "errors" in data and data["errors"]:
+                    # Allow partial data; retry on RATE_LIMITED
+                    errs = data["errors"]
+                    for e in errs:
+                        if e.get("type") == "RATE_LIMITED":
+                            self._sleep_until((self.graphql_reset or int(time.time()) + 60))
+                            break
+                    else:
+                        # No rate-limit error: log and return partial
+                        log.warning("GraphQL errors: %s", json.dumps(errs)[:400])
+                        return data
+                    continue
+                return data
+            except requests.RequestException as e:
+                last_err = e
+                log.warning("GraphQL network error: %s. Retry.", e)
+                time.sleep(backoff)
+                backoff = min(backoff * 2, 60)
+        raise RuntimeError(f"GraphQL failed after {max_retries} retries: {last_err}")
+
+    def rest(
+        self,
+        method: str,
+        path: str,
+        params: Optional[Dict[str, Any]] = None,
+        json_body: Optional[Dict[str, Any]] = None,
+        max_retries: int = 6,
+    ) -> requests.Response:
+        self._check_rate_and_wait("rest")
+        if path.startswith("http"):
+            url = path
+        else:
+            url = REST_BASE + path
+        backoff = 2
+        last_err = None
+        for attempt in range(max_retries):
+            try:
+                r = self.session.request(method, url, params = params, json = json_body, timeout = 120)
+                self.calls_rest += 1
+                rem = r.headers.get("X-RateLimit-Remaining")
+                rst = r.headers.get("X-RateLimit-Reset")
+                if rem is not None:
+                    try:
+                        self.rest_remaining = int(rem)
+                    except ValueError:
+                        pass
+                if rst is not None:
+                    try:
+                        self.rest_reset = int(rst)
+                    except ValueError:
+                        pass
+                if r.status_code in (502, 503, 504):
+                    log.warning("REST %s transient, retrying", r.status_code)
+                    time.sleep(backoff)
+                    backoff = min(backoff * 2, 60)
+                    continue
+                if self._is_auth_failure(r):
+                    self._raise_auth_error(r, "REST")
+                if r.status_code in (403, 429):
+                    retry_after = _retry_after_seconds(r.headers.get("Retry-After"))
+                    if retry_after is not None:
+                        log.warning("Secondary rate limit on REST. Sleep %ds.", retry_after)
+                        time.sleep(retry_after + 2)
+                        continue
+                    # Primary rate limit
+                    if self.rest_remaining == 0 and self.rest_reset:
+                        self._sleep_until(self.rest_reset)
+                        continue
+                    log.warning("REST 403/429, sleep 60")
+                    time.sleep(60)
+                    continue
+                return r
+            except requests.RequestException as e:
+                last_err = e
+                log.warning("REST network error: %s. Retry.", e)
+                time.sleep(backoff)
+                backoff = min(backoff * 2, 60)
+        raise RuntimeError(f"REST failed after {max_retries} retries: {last_err}")
+
+    def rest_paginate(
+        self,
+        path: str,
+        params: Optional[Dict[str, Any]] = None,
+        per_page: int = 100,
+    ) -> Iterator[dict]:
+        params = dict(params or {})
+        params.setdefault("per_page", per_page)
+        url = path
+        while True:
+            r = self.rest("GET", url, params = params if url == path else None)
+            if r.status_code != 200:
+                log.error("REST paginate got %s at %s: %s", r.status_code, url, r.text[:200])
+                return
+            items = r.json()
+            if isinstance(items, dict):
+                # Some endpoints wrap the list in an "items" field
+                items = items.get("items", [])
+            for it in items:
+                yield it
+            link = r.headers.get("Link", "")
+            nxt = None
+            for part in link.split(","):
+                if 'rel="next"' in part:
+                    nxt = part.split(";")[0].strip().strip("<>")
+                    break
+            if not nxt:
+                return
+            url = nxt
+            params = None
+
+    def rate_snapshot(self) -> Dict[str, Any]:
+        r = self.rest("GET", "/rate_limit")
+        if r.status_code == 200:
+            return r.json()
+        return {}

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/queries.py

@@ -0,0 +1,685 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""GraphQL queries for GitHub data scraping.
+
+GitHub's GraphQL rejects queries with unused fragments, so each query includes
+only the fragments it references.
+"""
+
+# ---- Fragments (raw strings, composed per query) ----
+F_ACTOR = """
+fragment ActorFields on Actor {
+  __typename
+  login
+  url
+  avatarUrl
+  ... on User { id databaseId name }
+  ... on Bot { id databaseId }
+  ... on Organization { id databaseId name }
+}
+"""
+
+F_LABEL = """
+fragment LabelFields on Label {
+  id
+  name
+  color
+  description
+  createdAt
+}
+"""
+
+F_TIMELINE = """
+fragment TimelineItem on IssueTimelineItems {
+  __typename
+  ... on Node { id }
+  ... on AddedToProjectEvent { createdAt actor { ...ActorFields } }
+  ... on AssignedEvent { createdAt actor { ...ActorFields } assignee { __typename ... on User { login } ... on Bot { login } } }
+  ... on ClosedEvent { createdAt actor { ...ActorFields } stateReason closer { __typename ... on Commit { oid url } ... on PullRequest { number url } } }
+  ... on CommentDeletedEvent { createdAt actor { ...ActorFields } }
+  ... on ConnectedEvent { createdAt actor { ...ActorFields } source { __typename ... on Issue { number url repository { nameWithOwner } } ... on PullRequest { number url repository { nameWithOwner } } } subject { __typename ... on Issue { number url } ... on PullRequest { number url } } }
+  ... on ConvertedNoteToIssueEvent { createdAt actor { ...ActorFields } }
+  ... on CrossReferencedEvent { createdAt actor { ...ActorFields } isCrossRepository willCloseTarget source { __typename ... on Issue { number url repository { nameWithOwner } title } ... on PullRequest { number url repository { nameWithOwner } title } } }
+  ... on DemilestonedEvent { createdAt actor { ...ActorFields } milestoneTitle }
+  ... on DisconnectedEvent { createdAt actor { ...ActorFields } subject { __typename ... on Issue { number url } ... on PullRequest { number url } } source { __typename ... on Issue { number url } ... on PullRequest { number url } } }
+  ... on IssueComment { id databaseId createdAt updatedAt author { ...ActorFields } body url reactionGroups { content reactors { totalCount } } }
+  ... on LabeledEvent { createdAt actor { ...ActorFields } label { name color } }
+  ... on LockedEvent { createdAt actor { ...ActorFields } lockReason }
+  ... on MarkedAsDuplicateEvent { createdAt actor { ...ActorFields } canonical { __typename ... on Issue { number url } ... on PullRequest { number url } } }
+  ... on MentionedEvent { createdAt actor { ...ActorFields } }
+  ... on MilestonedEvent { createdAt actor { ...ActorFields } milestoneTitle }
+  ... on MovedColumnsInProjectEvent { createdAt actor { ...ActorFields } }
+  ... on PinnedEvent { createdAt actor { ...ActorFields } }
+  ... on ReferencedEvent { createdAt actor { ...ActorFields } commit { oid url } commitRepository { nameWithOwner } }
+  ... on RemovedFromProjectEvent { createdAt actor { ...ActorFields } }
+  ... on RenamedTitleEvent { createdAt actor { ...ActorFields } previousTitle currentTitle }
+  ... on ReopenedEvent { createdAt actor { ...ActorFields } }
+  ... on SubscribedEvent { createdAt actor { ...ActorFields } }
+  ... on TransferredEvent { createdAt actor { ...ActorFields } fromRepository { nameWithOwner } }
+  ... on UnassignedEvent { createdAt actor { ...ActorFields } assignee { __typename ... on User { login } ... on Bot { login } } }
+  ... on UnlabeledEvent { createdAt actor { ...ActorFields } label { name color } }
+  ... on UnlockedEvent { createdAt actor { ...ActorFields } }
+  ... on UnmarkedAsDuplicateEvent { createdAt actor { ...ActorFields } }
+  ... on UnpinnedEvent { createdAt actor { ...ActorFields } }
+  ... on UnsubscribedEvent { createdAt actor { ...ActorFields } }
+  ... on UserBlockedEvent { createdAt actor { ...ActorFields } blockDuration }
+}
+"""
+
+F_PR_TIMELINE = """
+fragment PRTimelineItem on PullRequestTimelineItems {
+  __typename
+  ... on Node { id }
+  ... on AssignedEvent { createdAt actor { ...ActorFields } assignee { __typename ... on User { login } ... on Bot { login } } }
+  ... on AutoMergeDisabledEvent { createdAt actor { ...ActorFields } reason }
+  ... on AutoMergeEnabledEvent { createdAt actor { ...ActorFields } }
+  ... on AutoRebaseEnabledEvent { createdAt actor { ...ActorFields } }
+  ... on AutoSquashEnabledEvent { createdAt actor { ...ActorFields } }
+  ... on AutomaticBaseChangeFailedEvent { createdAt actor { ...ActorFields } oldBase newBase }
+  ... on AutomaticBaseChangeSucceededEvent { createdAt actor { ...ActorFields } oldBase newBase }
+  ... on BaseRefChangedEvent { createdAt actor { ...ActorFields } previousRefName currentRefName }
+  ... on BaseRefDeletedEvent { createdAt actor { ...ActorFields } baseRefName }
+  ... on BaseRefForcePushedEvent { createdAt actor { ...ActorFields } beforeCommit { oid } afterCommit { oid } ref { name } }
+  ... on ClosedEvent { createdAt actor { ...ActorFields } stateReason }
+  ... on CommentDeletedEvent { createdAt actor { ...ActorFields } }
+  ... on ConnectedEvent { createdAt actor { ...ActorFields } source { __typename ... on Issue { number url } ... on PullRequest { number url } } subject { __typename ... on Issue { number url } ... on PullRequest { number url } } }
+  ... on ConvertToDraftEvent { createdAt actor { ...ActorFields } }
+  ... on CrossReferencedEvent { createdAt actor { ...ActorFields } isCrossRepository willCloseTarget source { __typename ... on Issue { number url repository { nameWithOwner } title } ... on PullRequest { number url repository { nameWithOwner } title } } }
+  ... on DemilestonedEvent { createdAt actor { ...ActorFields } milestoneTitle }
+  ... on DeployedEvent { createdAt actor { ...ActorFields } }
+  ... on DeploymentEnvironmentChangedEvent { createdAt actor { ...ActorFields } }
+  ... on DisconnectedEvent { createdAt actor { ...ActorFields } subject { __typename ... on Issue { number url } ... on PullRequest { number url } } source { __typename ... on Issue { number url } ... on PullRequest { number url } } }
+  ... on HeadRefDeletedEvent { createdAt actor { ...ActorFields } headRefName }
+  ... on HeadRefForcePushedEvent { createdAt actor { ...ActorFields } beforeCommit { oid } afterCommit { oid } ref { name } }
+  ... on HeadRefRestoredEvent { createdAt actor { ...ActorFields } }
+  ... on IssueComment { id databaseId createdAt updatedAt author { ...ActorFields } body url reactionGroups { content reactors { totalCount } } }
+  ... on LabeledEvent { createdAt actor { ...ActorFields } label { name color } }
+  ... on LockedEvent { createdAt actor { ...ActorFields } lockReason }
+  ... on MarkedAsDuplicateEvent { createdAt actor { ...ActorFields } canonical { __typename ... on Issue { number url } ... on PullRequest { number url } } }
+  ... on MentionedEvent { createdAt actor { ...ActorFields } }
+  ... on MergedEvent { createdAt actor { ...ActorFields } commit { oid url } mergeRefName }
+  ... on MilestonedEvent { createdAt actor { ...ActorFields } milestoneTitle }
+  ... on MovedColumnsInProjectEvent { createdAt actor { ...ActorFields } }
+  ... on PinnedEvent { createdAt actor { ...ActorFields } }
+  ... on PullRequestCommit { commit { oid url message author { user { login } date } committedDate } }
+  ... on PullRequestCommitCommentThread { commit { oid } }
+  ... on PullRequestReview { id databaseId createdAt submittedAt author { ...ActorFields } body state url reactionGroups { content reactors { totalCount } } }
+  ... on PullRequestReviewThread { id isResolved isOutdated path line diffSide }
+  ... on PullRequestRevisionMarker { createdAt lastSeenCommit { oid } }
+  ... on ReadyForReviewEvent { createdAt actor { ...ActorFields } }
+  ... on ReferencedEvent { createdAt actor { ...ActorFields } commit { oid url } commitRepository { nameWithOwner } }
+  ... on RenamedTitleEvent { createdAt actor { ...ActorFields } previousTitle currentTitle }
+  ... on ReopenedEvent { createdAt actor { ...ActorFields } }
+  ... on ReviewDismissedEvent { createdAt actor { ...ActorFields } dismissalMessage previousReviewState }
+  ... on ReviewRequestRemovedEvent { createdAt actor { ...ActorFields } requestedReviewer { __typename ... on User { login } ... on Team { name } } }
+  ... on ReviewRequestedEvent { createdAt actor { ...ActorFields } requestedReviewer { __typename ... on User { login } ... on Team { name } } }
+  ... on SubscribedEvent { createdAt actor { ...ActorFields } }
+  ... on TransferredEvent { createdAt actor { ...ActorFields } fromRepository { nameWithOwner } }
+  ... on UnassignedEvent { createdAt actor { ...ActorFields } assignee { __typename ... on User { login } ... on Bot { login } } }
+  ... on UnlabeledEvent { createdAt actor { ...ActorFields } label { name color } }
+  ... on UnlockedEvent { createdAt actor { ...ActorFields } }
+  ... on UnmarkedAsDuplicateEvent { createdAt actor { ...ActorFields } }
+  ... on UnpinnedEvent { createdAt actor { ...ActorFields } }
+  ... on UnsubscribedEvent { createdAt actor { ...ActorFields } }
+  ... on UserBlockedEvent { createdAt actor { ...ActorFields } blockDuration }
+}
+"""
+
+
+def _q(parts: list[str], body: str) -> str:
+    return "\n".join(parts + [body])
+
+
+ISSUES_PAGE_QUERY = _q(
+    [F_ACTOR, F_LABEL, F_TIMELINE],
+    """
+query IssuesPage($owner: String!, $name: String!, $first: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    issues(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) {
+      pageInfo { hasNextPage endCursor }
+      totalCount
+      nodes {
+        id databaseId number title body state stateReason
+        createdAt updatedAt closedAt
+        url
+        author { ...ActorFields }
+        editor { ...ActorFields }
+        labels(first: 50) { nodes { ...LabelFields } }
+        assignees(first: 20) { nodes { login id } }
+        milestone { title number state dueOn }
+        reactionGroups { content reactors { totalCount } }
+        comments(first: 100) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            id databaseId createdAt updatedAt url body
+            author { ...ActorFields }
+            editor { ...ActorFields }
+            reactionGroups { content reactors { totalCount } }
+          }
+        }
+        timelineItems(first: 100) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes { ...TimelineItem }
+        }
+        trackedInIssues(first: 20) { totalCount nodes { number url repository { nameWithOwner } } }
+        trackedIssues(first: 20) { totalCount nodes { number url repository { nameWithOwner } } }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+PRS_PAGE_QUERY = _q(
+    [F_ACTOR, F_LABEL, F_PR_TIMELINE],
+    """
+query PRsPage($owner: String!, $name: String!, $first: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    pullRequests(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) {
+      pageInfo { hasNextPage endCursor }
+      totalCount
+      nodes {
+        id databaseId number title body state isDraft
+        createdAt updatedAt closedAt mergedAt
+        url
+        headRefName headRefOid
+        baseRefName baseRefOid
+        additions deletions changedFiles
+        mergeable merged mergeStateStatus
+        author { ...ActorFields }
+        editor { ...ActorFields }
+        mergedBy { ...ActorFields }
+        labels(first: 50) { nodes { ...LabelFields } }
+        assignees(first: 20) { nodes { login id } }
+        milestone { title number state dueOn }
+        reactionGroups { content reactors { totalCount } }
+        closingIssuesReferences(first: 20) { totalCount nodes { number url repository { nameWithOwner } title } }
+        comments(first: 100) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            id databaseId createdAt updatedAt url body
+            author { ...ActorFields }
+            editor { ...ActorFields }
+            reactionGroups { content reactors { totalCount } }
+          }
+        }
+        reviewThreads(first: 50) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            id isResolved isOutdated path line diffSide
+            comments(first: 50) {
+              totalCount
+              pageInfo { hasNextPage endCursor }
+              nodes {
+                id databaseId createdAt updatedAt url body path diffHunk
+                author { ...ActorFields }
+                editor { ...ActorFields }
+                position originalPosition line originalLine
+                commit { oid }
+                reactionGroups { content reactors { totalCount } }
+              }
+            }
+          }
+        }
+        reviews(first: 50) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            id databaseId state createdAt submittedAt body url
+            author { ...ActorFields }
+            reactionGroups { content reactors { totalCount } }
+          }
+        }
+        commits(first: 100) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            commit {
+              oid
+              message
+              messageHeadline
+              committedDate
+              authoredDate
+              author { name email user { login } date }
+              committer { name email user { login } date }
+              additions deletions changedFilesIfAvailable
+              parents(first: 3) { nodes { oid } }
+            }
+          }
+        }
+        files(first: 100) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            path additions deletions changeType
+          }
+        }
+        timelineItems(first: 100) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes { ...PRTimelineItem }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+PRS_PAGE_QUERY_LIGHT = _q(
+    [F_ACTOR, F_LABEL],
+    """
+query PRsPageLight($owner: String!, $name: String!, $first: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    pullRequests(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) {
+      pageInfo { hasNextPage endCursor }
+      totalCount
+      nodes {
+        id databaseId number title body state isDraft
+        createdAt updatedAt closedAt mergedAt
+        url
+        author { ...ActorFields }
+        labels(first: 50) { nodes { ...LabelFields } }
+        comments(first: 30) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            id databaseId createdAt updatedAt url body
+            author { ...ActorFields }
+          }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+ISSUES_PAGE_QUERY_LIGHT = _q(
+    [F_ACTOR, F_LABEL],
+    """
+query IssuesPageLight($owner: String!, $name: String!, $first: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    issues(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) {
+      pageInfo { hasNextPage endCursor }
+      totalCount
+      nodes {
+        id databaseId number title body state
+        createdAt updatedAt closedAt
+        url
+        author { ...ActorFields }
+        labels(first: 50) { nodes { ...LabelFields } }
+        comments(first: 30) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            id databaseId createdAt updatedAt url body
+            author { ...ActorFields }
+          }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+ISSUE_COMMENTS_QUERY = _q(
+    [F_ACTOR],
+    """
+query IssueComments($owner: String!, $name: String!, $number: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    issueOrPullRequest(number: $number) {
+      __typename
+      ... on Issue {
+        comments(first: 100, after: $after) {
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            id databaseId createdAt updatedAt url body
+            author { ...ActorFields }
+            editor { ...ActorFields }
+            reactionGroups { content reactors { totalCount } }
+          }
+        }
+      }
+      ... on PullRequest {
+        comments(first: 100, after: $after) {
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            id databaseId createdAt updatedAt url body
+            author { ...ActorFields }
+            editor { ...ActorFields }
+            reactionGroups { content reactors { totalCount } }
+          }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+ISSUE_TIMELINE_QUERY = _q(
+    [F_ACTOR, F_TIMELINE],
+    """
+query IssueTimeline($owner: String!, $name: String!, $number: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    issue(number: $number) {
+      timelineItems(first: 100, after: $after) {
+        pageInfo { hasNextPage endCursor }
+        nodes { ...TimelineItem }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+PR_TIMELINE_QUERY = _q(
+    [F_ACTOR, F_PR_TIMELINE],
+    """
+query PRTimeline($owner: String!, $name: String!, $number: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    pullRequest(number: $number) {
+      timelineItems(first: 100, after: $after) {
+        pageInfo { hasNextPage endCursor }
+        nodes { ...PRTimelineItem }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+PR_COMMITS_QUERY = """
+query PRCommits($owner: String!, $name: String!, $number: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    pullRequest(number: $number) {
+      commits(first: 100, after: $after) {
+        pageInfo { hasNextPage endCursor }
+        nodes {
+          commit {
+            oid message messageHeadline committedDate authoredDate
+            author { name email user { login } date }
+            committer { name email user { login } date }
+            additions deletions changedFilesIfAvailable
+            parents(first: 3) { nodes { oid } }
+          }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+"""
+
+PR_FILES_QUERY = """
+query PRFiles($owner: String!, $name: String!, $number: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    pullRequest(number: $number) {
+      files(first: 100, after: $after) {
+        pageInfo { hasNextPage endCursor }
+        nodes { path additions deletions changeType }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+"""
+
+PR_REVIEW_THREADS_QUERY = _q(
+    [F_ACTOR],
+    """
+query PRReviewThreads($owner: String!, $name: String!, $number: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    pullRequest(number: $number) {
+      reviewThreads(first: 50, after: $after) {
+        pageInfo { hasNextPage endCursor }
+        nodes {
+          id isResolved isOutdated path line diffSide
+          comments(first: 50) {
+            totalCount
+            nodes {
+              id databaseId createdAt updatedAt url body path diffHunk
+              author { ...ActorFields }
+              editor { ...ActorFields }
+              position originalPosition line originalLine
+              commit { oid }
+              reactionGroups { content reactors { totalCount } }
+            }
+          }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+DISCUSSIONS_PAGE_QUERY = _q(
+    [F_ACTOR, F_LABEL],
+    """
+query DiscussionsPage($owner: String!, $name: String!, $first: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    discussions(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) {
+      pageInfo { hasNextPage endCursor }
+      totalCount
+      nodes {
+        id databaseId number title body
+        createdAt updatedAt url
+        author { ...ActorFields }
+        editor { ...ActorFields }
+        locked
+        answerChosenAt
+        closed closedAt
+        category { id name emoji description isAnswerable }
+        labels(first: 30) { nodes { ...LabelFields } }
+        upvoteCount
+        answer { id databaseId body author { ...ActorFields } createdAt url }
+        reactionGroups { content reactors { totalCount } }
+        comments(first: 50) {
+          totalCount
+          pageInfo { hasNextPage endCursor }
+          nodes {
+            id databaseId body createdAt updatedAt url
+            author { ...ActorFields }
+            editor { ...ActorFields }
+            upvoteCount
+            isAnswer
+            reactionGroups { content reactors { totalCount } }
+            replies(first: 50) {
+              totalCount
+              pageInfo { hasNextPage endCursor }
+              nodes {
+                id databaseId body createdAt updatedAt url
+                author { ...ActorFields }
+                editor { ...ActorFields }
+                reactionGroups { content reactors { totalCount } }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+DISCUSSION_COMMENTS_QUERY = _q(
+    [F_ACTOR],
+    """
+query DiscussionComments($owner: String!, $name: String!, $number: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    discussion(number: $number) {
+      comments(first: 50, after: $after) {
+        pageInfo { hasNextPage endCursor }
+        nodes {
+          id databaseId body createdAt updatedAt url
+          author { ...ActorFields }
+          editor { ...ActorFields }
+          upvoteCount
+          isAnswer
+          reactionGroups { content reactors { totalCount } }
+          replies(first: 50) {
+            totalCount
+            nodes {
+              id databaseId body createdAt updatedAt url
+              author { ...ActorFields }
+              editor { ...ActorFields }
+              reactionGroups { content reactors { totalCount } }
+            }
+          }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+DISCUSSION_REPLIES_QUERY = _q(
+    [F_ACTOR],
+    """
+query DiscussionReplies($commentId: ID!, $after: String) {
+  node(id: $commentId) {
+    ... on DiscussionComment {
+      replies(first: 50, after: $after) {
+        pageInfo { hasNextPage endCursor }
+        nodes {
+          id databaseId body createdAt updatedAt url
+          author { ...ActorFields }
+          editor { ...ActorFields }
+          reactionGroups { content reactors { totalCount } }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+COMMITS_PAGE_QUERY = """
+query CommitsPage($owner: String!, $name: String!, $first: Int!, $after: String, $branch: String!) {
+  repository(owner: $owner, name: $name) {
+    ref(qualifiedName: $branch) {
+      target {
+        ... on Commit {
+          history(first: $first, after: $after) {
+            pageInfo { hasNextPage endCursor }
+            totalCount
+            nodes {
+              oid
+              message
+              messageHeadline
+              committedDate
+              authoredDate
+              url
+              additions deletions changedFilesIfAvailable
+              author { name email date user { login id } }
+              committer { name email date user { login id } }
+              parents(first: 3) { nodes { oid } }
+              associatedPullRequests(first: 5) { nodes { number url state } }
+            }
+          }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+"""
+
+RELEASES_QUERY = _q(
+    [F_ACTOR],
+    """
+query Releases($owner: String!, $name: String!, $first: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    releases(first: $first, after: $after, orderBy: {field: CREATED_AT, direction: ASC}) {
+      pageInfo { hasNextPage endCursor }
+      nodes {
+        id databaseId name tagName description
+        createdAt publishedAt updatedAt
+        isDraft isPrerelease isLatest
+        url
+        author { ...ActorFields }
+        tagCommit { oid url }
+        reactionGroups { content reactors { totalCount } }
+        releaseAssets(first: 50) {
+          nodes { name contentType size downloadUrl createdAt updatedAt }
+        }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+LABELS_QUERY = _q(
+    [F_LABEL],
+    """
+query LabelsList($owner: String!, $name: String!, $first: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    labels(first: $first, after: $after) {
+      pageInfo { hasNextPage endCursor }
+      nodes { ...LabelFields }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+""",
+)
+
+MILESTONES_QUERY = """
+query Milestones($owner: String!, $name: String!, $first: Int!, $after: String) {
+  repository(owner: $owner, name: $name) {
+    milestones(first: $first, after: $after) {
+      pageInfo { hasNextPage endCursor }
+      nodes {
+        id number title description state
+        createdAt updatedAt closedAt dueOn
+        creator { login }
+      }
+    }
+  }
+  rateLimit { cost remaining resetAt }
+}
+"""
+
+REPO_META_QUERY = """
+query RepoMeta($owner: String!, $name: String!) {
+  repository(owner: $owner, name: $name) {
+    id databaseId name nameWithOwner description url
+    createdAt updatedAt pushedAt
+    isArchived isDisabled isFork isPrivate
+    primaryLanguage { name }
+    languages(first: 20, orderBy: {field: SIZE, direction: DESC}) {
+      edges { size node { name } }
+      totalSize
+    }
+    stargazerCount forkCount watchers { totalCount }
+    diskUsage
+    licenseInfo { key name }
+    homepageUrl
+    defaultBranchRef { name }
+  }
+  rateLimit { cost remaining resetAt }
+}
+"""

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py

@@ -0,0 +1,705 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Scraper orchestration: issues, PRs, discussions, commits, releases, etc.
+
+Resumable via state file. Writes JSONL shards under data/{repo}/{resource}.jsonl.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+# Allow running as a module or script
+THIS_DIR = Path(__file__).resolve().parent
+if str(THIS_DIR) not in sys.path:
+    sys.path.insert(0, str(THIS_DIR))
+
+from gh_client import GitHubClient
+from state_store import JsonlWriter, StateStore
+import queries as Q
+
+log = logging.getLogger("scraper")
+
+
+def ts() -> str:
+    return time.strftime("%Y-%m-%d %H:%M:%S")
+
+
+class RepoScraper:
+    def __init__(
+        self,
+        owner: str,
+        name: str,
+        base_dir: Path,
+        client: GitHubClient,
+        trial_limits: Optional[Dict[str, int]] = None,
+        light: bool = False,
+    ):
+        self.owner = owner
+        self.name = name
+        self.base_dir = base_dir
+        self.client = client
+        self.trial_limits = trial_limits or {}
+        # light=True uses trimmed GraphQL queries (no reviewThreads/reviews/
+        # commits/timelineItems/files) so PR pages can be larger without
+        # hitting GitHub's node-count ceiling.
+        self.light = light
+        self.repo_dir = base_dir / f"{owner}__{name}"
+        self.repo_dir.mkdir(parents = True, exist_ok = True)
+        self.state = StateStore(base_dir / "state" / f"{owner}__{name}.json")
+
+        # Writers
+        self.writers: Dict[str, JsonlWriter] = {}
+        for key in (
+            "issues",
+            "pull_requests",
+            "discussions",
+            "commits",
+            "releases",
+            "labels",
+            "milestones",
+            "pr_extra_comments",
+            "pr_extra_timeline",
+            "pr_extra_reviews",
+            "issue_extra_comments",
+            "issue_extra_timeline",
+            "discussion_extra_comments",
+            "discussion_extra_replies",
+            "repo_meta",
+        ):
+            self.writers[key] = JsonlWriter(self.repo_dir / f"{key}.jsonl")
+
+    # ----- helpers -----
+    def _trial_stop(self, key: str, counter: int) -> bool:
+        lim = self.trial_limits.get(key)
+        if lim is None:
+            return False
+        return counter >= lim
+
+    def _log_rate(self, where: str, data: Dict[str, Any]) -> None:
+        rl = data.get("data", {}).get("rateLimit") if isinstance(data.get("data"), dict) else None
+        if rl:
+            log.debug(
+                "[%s] rate cost=%s remaining=%s resetAt=%s",
+                where,
+                rl.get("cost"),
+                rl.get("remaining"),
+                rl.get("resetAt"),
+            )
+
+    # ----- repo meta -----
+    def scrape_repo_meta(self) -> Dict[str, Any]:
+        data = self.client.graphql(Q.REPO_META_QUERY, {"owner": self.owner, "name": self.name})
+        self._log_rate("repo_meta", data)
+        repo = data.get("data", {}).get("repository") or {}
+        repo["_fetchedAt"] = ts()
+        self.writers["repo_meta"].write(repo)
+        return repo
+
+    # ----- issues -----
+    def scrape_issues(self) -> int:
+        key = "issues"
+        cursor = self.state.get(f"{key}_cursor")
+        done = self.state.get(f"{key}_done", False)
+        if done:
+            log.info("%s/%s issues already complete", self.owner, self.name)
+            return 0
+        total_new = 0
+        page = 0
+        # Light query skips heavy nested fields; safe at 50/page. Clamp by
+        # trial_limit so limit=1 asks for first:1, not a full 50-item page.
+        page_cap = 50 if self.light else 15
+        trial_cap = self.trial_limits.get(key)
+        per_page = min(page_cap, trial_cap) if trial_cap and trial_cap > 0 else page_cap
+        while True:
+            page += 1
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "first": per_page,
+                "after": cursor,
+            }
+            query = Q.ISSUES_PAGE_QUERY_LIGHT if self.light else Q.ISSUES_PAGE_QUERY
+            data = self.client.graphql(query, vars_)
+            self._log_rate("issues", data)
+            repo = (data.get("data") or {}).get("repository") or {}
+            issues = repo.get("issues") or {}
+            nodes = issues.get("nodes") or []
+            for it in nodes:
+                it["_owner"] = self.owner
+                it["_repo"] = self.name
+                it["_fetchedAt"] = ts()
+                if not self.light:
+                    if it.get("comments", {}).get("pageInfo", {}).get("hasNextPage"):
+                        self._paginate_issue_comments(
+                            it["number"], it["comments"]["pageInfo"]["endCursor"]
+                        )
+                    if it.get("timelineItems", {}).get("pageInfo", {}).get("hasNextPage"):
+                        self._paginate_issue_timeline(
+                            it["number"],
+                            it["timelineItems"]["pageInfo"]["endCursor"],
+                        )
+                if self.writers[key].write(it):
+                    total_new += 1
+            info = issues.get("pageInfo") or {}
+            cursor = info.get("endCursor")
+            self.state.set(f"{key}_cursor", cursor)
+            log.info(
+                "[%s/%s] issues page %d (+%d) cursor=%s remaining=%s",
+                self.owner,
+                self.name,
+                page,
+                len(nodes),
+                str(cursor)[:20],
+                self.client.graphql_remaining,
+            )
+            if self._trial_stop(key, total_new):
+                log.info("Trial limit reached for issues (%d)", total_new)
+                return total_new
+            if not info.get("hasNextPage"):
+                self.state.set(f"{key}_done", True)
+                break
+        return total_new
+
+    def _paginate_issue_comments(self, number: int, after: str) -> None:
+        cur = after
+        while cur:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "number": number,
+                "after": cur,
+            }
+            data = self.client.graphql(Q.ISSUE_COMMENTS_QUERY, vars_)
+            item = ((data.get("data") or {}).get("repository") or {}).get(
+                "issueOrPullRequest"
+            ) or {}
+            comments = item.get("comments") or {}
+            for c in comments.get("nodes") or []:
+                c["_owner"] = self.owner
+                c["_repo"] = self.name
+                c["_issueNumber"] = number
+                self.writers["issue_extra_comments"].write(c)
+            info = comments.get("pageInfo") or {}
+            cur = info.get("endCursor") if info.get("hasNextPage") else None
+
+    def _paginate_issue_timeline(self, number: int, after: str) -> None:
+        cur = after
+        while cur:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "number": number,
+                "after": cur,
+            }
+            data = self.client.graphql(Q.ISSUE_TIMELINE_QUERY, vars_)
+            item = ((data.get("data") or {}).get("repository") or {}).get("issue") or {}
+            tl = item.get("timelineItems") or {}
+            for ev in tl.get("nodes") or []:
+                ev["_owner"] = self.owner
+                ev["_repo"] = self.name
+                ev["_issueNumber"] = number
+                self.writers["issue_extra_timeline"].write(ev)
+            info = tl.get("pageInfo") or {}
+            cur = info.get("endCursor") if info.get("hasNextPage") else None
+
+    # ----- PRs -----
+    def scrape_prs(self) -> int:
+        key = "pull_requests"
+        cursor = self.state.get(f"{key}_cursor")
+        done = self.state.get(f"{key}_done", False)
+        if done:
+            log.info("%s/%s PRs already complete", self.owner, self.name)
+            return 0
+        total_new = 0
+        page = 0
+        # Heavy nested PR query caps at 3/page (GitHub node-count ceiling);
+        # light query skips nested fields and goes to 25/page. Clamp by
+        # trial_limit so limit=1 does not fetch a whole 25-item page.
+        page_cap = 25 if self.light else 3
+        trial_cap = self.trial_limits.get(key)
+        per_page = min(page_cap, trial_cap) if trial_cap and trial_cap > 0 else page_cap
+        while True:
+            page += 1
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "first": per_page,
+                "after": cursor,
+            }
+            query = Q.PRS_PAGE_QUERY_LIGHT if self.light else Q.PRS_PAGE_QUERY
+            data = self.client.graphql(query, vars_)
+            self._log_rate("prs", data)
+            repo = (data.get("data") or {}).get("repository") or {}
+            prs = repo.get("pullRequests") or {}
+            nodes = prs.get("nodes") or []
+            for pr in nodes:
+                pr["_owner"] = self.owner
+                pr["_repo"] = self.name
+                pr["_fetchedAt"] = ts()
+                num = pr["number"]
+                if not self.light:
+                    if pr.get("comments", {}).get("pageInfo", {}).get("hasNextPage"):
+                        self._paginate_pr_comments(num, pr["comments"]["pageInfo"]["endCursor"])
+                    if pr.get("timelineItems", {}).get("pageInfo", {}).get("hasNextPage"):
+                        self._paginate_pr_timeline(
+                            num, pr["timelineItems"]["pageInfo"]["endCursor"]
+                        )
+                    if pr.get("commits", {}).get("pageInfo", {}).get("hasNextPage"):
+                        self._paginate_pr_commits(num, pr["commits"]["pageInfo"]["endCursor"])
+                    if pr.get("files", {}).get("pageInfo", {}).get("hasNextPage"):
+                        self._paginate_pr_files(num, pr["files"]["pageInfo"]["endCursor"])
+                    if pr.get("reviewThreads", {}).get("pageInfo", {}).get("hasNextPage"):
+                        self._paginate_pr_review_threads(
+                            num, pr["reviewThreads"]["pageInfo"]["endCursor"]
+                        )
+                if self.writers[key].write(pr):
+                    total_new += 1
+            info = prs.get("pageInfo") or {}
+            cursor = info.get("endCursor")
+            self.state.set(f"{key}_cursor", cursor)
+            log.info(
+                "[%s/%s] PRs page %d (+%d) cursor=%s remaining=%s",
+                self.owner,
+                self.name,
+                page,
+                len(nodes),
+                str(cursor)[:20],
+                self.client.graphql_remaining,
+            )
+            if self._trial_stop(key, total_new):
+                log.info("Trial limit reached for PRs (%d)", total_new)
+                return total_new
+            if not info.get("hasNextPage"):
+                self.state.set(f"{key}_done", True)
+                break
+        return total_new
+
+    def _paginate_pr_comments(self, number: int, after: str) -> None:
+        cur = after
+        while cur:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "number": number,
+                "after": cur,
+            }
+            data = self.client.graphql(Q.ISSUE_COMMENTS_QUERY, vars_)
+            item = ((data.get("data") or {}).get("repository") or {}).get(
+                "issueOrPullRequest"
+            ) or {}
+            comments = item.get("comments") or {}
+            for c in comments.get("nodes") or []:
+                c["_owner"] = self.owner
+                c["_repo"] = self.name
+                c["_prNumber"] = number
+                self.writers["pr_extra_comments"].write(c)
+            info = comments.get("pageInfo") or {}
+            cur = info.get("endCursor") if info.get("hasNextPage") else None
+
+    def _paginate_pr_timeline(self, number: int, after: str) -> None:
+        cur = after
+        while cur:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "number": number,
+                "after": cur,
+            }
+            data = self.client.graphql(Q.PR_TIMELINE_QUERY, vars_)
+            item = ((data.get("data") or {}).get("repository") or {}).get("pullRequest") or {}
+            tl = item.get("timelineItems") or {}
+            for ev in tl.get("nodes") or []:
+                ev["_owner"] = self.owner
+                ev["_repo"] = self.name
+                ev["_prNumber"] = number
+                self.writers["pr_extra_timeline"].write(ev)
+            info = tl.get("pageInfo") or {}
+            cur = info.get("endCursor") if info.get("hasNextPage") else None
+
+    def _paginate_pr_commits(self, number: int, after: str) -> None:
+        cur = after
+        out_key = "pr_extra_commits"
+        if out_key not in self.writers:
+            self.writers[out_key] = JsonlWriter(self.repo_dir / f"{out_key}.jsonl")
+        while cur:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "number": number,
+                "after": cur,
+            }
+            data = self.client.graphql(Q.PR_COMMITS_QUERY, vars_)
+            item = ((data.get("data") or {}).get("repository") or {}).get("pullRequest") or {}
+            cc = item.get("commits") or {}
+            for c in cc.get("nodes") or []:
+                c["_owner"] = self.owner
+                c["_repo"] = self.name
+                c["_prNumber"] = number
+                self.writers[out_key].write(c)
+            info = cc.get("pageInfo") or {}
+            cur = info.get("endCursor") if info.get("hasNextPage") else None
+
+    def _paginate_pr_files(self, number: int, after: str) -> None:
+        cur = after
+        out_key = "pr_extra_files"
+        if out_key not in self.writers:
+            self.writers[out_key] = JsonlWriter(self.repo_dir / f"{out_key}.jsonl")
+        while cur:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "number": number,
+                "after": cur,
+            }
+            data = self.client.graphql(Q.PR_FILES_QUERY, vars_)
+            item = ((data.get("data") or {}).get("repository") or {}).get("pullRequest") or {}
+            ff = item.get("files") or {}
+            for f in ff.get("nodes") or []:
+                f["_owner"] = self.owner
+                f["_repo"] = self.name
+                f["_prNumber"] = number
+                # files have no id; synthesize one
+                f["_syntheticId"] = f"{self.owner}/{self.name}#{number}:{f.get('path')}"
+                self.writers[out_key].write(f)
+            info = ff.get("pageInfo") or {}
+            cur = info.get("endCursor") if info.get("hasNextPage") else None
+
+    def _paginate_pr_review_threads(self, number: int, after: str) -> None:
+        cur = after
+        out_key = "pr_extra_review_threads"
+        if out_key not in self.writers:
+            self.writers[out_key] = JsonlWriter(self.repo_dir / f"{out_key}.jsonl")
+        while cur:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "number": number,
+                "after": cur,
+            }
+            data = self.client.graphql(Q.PR_REVIEW_THREADS_QUERY, vars_)
+            item = ((data.get("data") or {}).get("repository") or {}).get("pullRequest") or {}
+            rt = item.get("reviewThreads") or {}
+            for th in rt.get("nodes") or []:
+                th["_owner"] = self.owner
+                th["_repo"] = self.name
+                th["_prNumber"] = number
+                self.writers[out_key].write(th)
+            info = rt.get("pageInfo") or {}
+            cur = info.get("endCursor") if info.get("hasNextPage") else None
+
+    # ----- Discussions -----
+    def scrape_discussions(self) -> int:
+        key = "discussions"
+        cursor = self.state.get(f"{key}_cursor")
+        done = self.state.get(f"{key}_done", False)
+        if done:
+            log.info("%s/%s discussions already complete", self.owner, self.name)
+            return 0
+        total_new = 0
+        page = 0
+        per_page = 15
+        while True:
+            page += 1
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "first": per_page,
+                "after": cursor,
+            }
+            data = self.client.graphql(Q.DISCUSSIONS_PAGE_QUERY, vars_)
+            self._log_rate("discussions", data)
+            repo = (data.get("data") or {}).get("repository") or {}
+            dd = repo.get("discussions") or {}
+            nodes = dd.get("nodes") or []
+            for d in nodes:
+                d["_owner"] = self.owner
+                d["_repo"] = self.name
+                d["_fetchedAt"] = ts()
+                num = d["number"]
+                if d.get("comments", {}).get("pageInfo", {}).get("hasNextPage"):
+                    self._paginate_discussion_comments(num, d["comments"]["pageInfo"]["endCursor"])
+                # paginate replies per comment if needed
+                for c in d.get("comments", {}).get("nodes", []) or []:
+                    if c.get("replies", {}).get("pageInfo", {}).get("hasNextPage"):
+                        self._paginate_discussion_replies(
+                            c["id"], c["replies"]["pageInfo"]["endCursor"], num
+                        )
+                if self.writers[key].write(d):
+                    total_new += 1
+            info = dd.get("pageInfo") or {}
+            cursor = info.get("endCursor")
+            self.state.set(f"{key}_cursor", cursor)
+            log.info(
+                "[%s/%s] discussions page %d (+%d) cursor=%s remaining=%s",
+                self.owner,
+                self.name,
+                page,
+                len(nodes),
+                str(cursor)[:20],
+                self.client.graphql_remaining,
+            )
+            if self._trial_stop(key, total_new):
+                return total_new
+            if not info.get("hasNextPage"):
+                self.state.set(f"{key}_done", True)
+                break
+        return total_new
+
+    def _paginate_discussion_comments(self, number: int, after: str) -> None:
+        cur = after
+        while cur:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "number": number,
+                "after": cur,
+            }
+            data = self.client.graphql(Q.DISCUSSION_COMMENTS_QUERY, vars_)
+            disc = ((data.get("data") or {}).get("repository") or {}).get("discussion") or {}
+            cc = disc.get("comments") or {}
+            for c in cc.get("nodes") or []:
+                c["_owner"] = self.owner
+                c["_repo"] = self.name
+                c["_discussionNumber"] = number
+                self.writers["discussion_extra_comments"].write(c)
+            info = cc.get("pageInfo") or {}
+            cur = info.get("endCursor") if info.get("hasNextPage") else None
+
+    def _paginate_discussion_replies(self, comment_id: str, after: str, disc_number: int) -> None:
+        cur = after
+        while cur:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "commentId": comment_id,
+                "after": cur,
+            }
+            data = self.client.graphql(Q.DISCUSSION_REPLIES_QUERY, vars_)
+            node = (data.get("data") or {}).get("node") or {}
+            replies = node.get("replies") or {}
+            for r in replies.get("nodes") or []:
+                r["_owner"] = self.owner
+                r["_repo"] = self.name
+                r["_discussionNumber"] = disc_number
+                r["_commentId"] = comment_id
+                self.writers["discussion_extra_replies"].write(r)
+            info = replies.get("pageInfo") or {}
+            cur = info.get("endCursor") if info.get("hasNextPage") else None
+
+    # ----- Commits -----
+    def scrape_commits(self, branch: str = "refs/heads/main") -> int:
+        key = "commits"
+        cursor = self.state.get(f"{key}_cursor")
+        done = self.state.get(f"{key}_done", False)
+        if done:
+            return 0
+        total_new = 0
+        page = 0
+        page_cap = 100
+        trial_cap = self.trial_limits.get(key)
+        per_page = min(page_cap, trial_cap) if trial_cap and trial_cap > 0 else page_cap
+        while True:
+            page += 1
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "first": per_page,
+                "after": cursor,
+                "branch": branch,
+            }
+            data = self.client.graphql(Q.COMMITS_PAGE_QUERY, vars_)
+            self._log_rate("commits", data)
+            ref = ((data.get("data") or {}).get("repository") or {}).get("ref") or {}
+            tgt = ref.get("target") or {}
+            hist = tgt.get("history") or {}
+            nodes = hist.get("nodes") or []
+            for c in nodes:
+                c["_owner"] = self.owner
+                c["_repo"] = self.name
+                c["_fetchedAt"] = ts()
+                if self.writers[key].write(c):
+                    total_new += 1
+            info = hist.get("pageInfo") or {}
+            cursor = info.get("endCursor")
+            self.state.set(f"{key}_cursor", cursor)
+            log.info(
+                "[%s/%s] commits page %d (+%d) remaining=%s",
+                self.owner,
+                self.name,
+                page,
+                len(nodes),
+                self.client.graphql_remaining,
+            )
+            if self._trial_stop(key, total_new):
+                return total_new
+            if not info.get("hasNextPage"):
+                self.state.set(f"{key}_done", True)
+                break
+        return total_new
+
+    # ----- Releases/Labels/Milestones -----
+    def scrape_releases(self) -> int:
+        return self._scrape_simple("releases", Q.RELEASES_QUERY, "releases")
+
+    def scrape_labels(self) -> int:
+        return self._scrape_simple("labels", Q.LABELS_QUERY, "labels")
+
+    def scrape_milestones(self) -> int:
+        return self._scrape_simple("milestones", Q.MILESTONES_QUERY, "milestones")
+
+    def _scrape_simple(self, key: str, query: str, field: str) -> int:
+        cursor = self.state.get(f"{key}_cursor")
+        done = self.state.get(f"{key}_done", False)
+        if done:
+            return 0
+        total_new = 0
+        while True:
+            vars_ = {
+                "owner": self.owner,
+                "name": self.name,
+                "first": 50,
+                "after": cursor,
+            }
+            data = self.client.graphql(query, vars_)
+            repo = (data.get("data") or {}).get("repository") or {}
+            col = repo.get(field) or {}
+            for it in col.get("nodes") or []:
+                it["_owner"] = self.owner
+                it["_repo"] = self.name
+                it["_fetchedAt"] = ts()
+                if self.writers[key].write(it):
+                    total_new += 1
+            info = col.get("pageInfo") or {}
+            cursor = info.get("endCursor")
+            self.state.set(f"{key}_cursor", cursor)
+            if self._trial_stop(key, total_new):
+                return total_new
+            if not info.get("hasNextPage"):
+                self.state.set(f"{key}_done", True)
+                break
+        log.info("[%s/%s] %s done +%d", self.owner, self.name, key, total_new)
+        return total_new
+
+    def close(self) -> None:
+        for w in self.writers.values():
+            try:
+                w.close()
+            except Exception:
+                pass
+
+
+def setup_logging(log_file: Path) -> None:
+    log_file.parent.mkdir(parents = True, exist_ok = True)
+    fmt = "%(asctime)s %(levelname)s [%(name)s] %(message)s"
+    handlers = [
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler(log_file, mode = "a", encoding = "utf-8"),
+    ]
+    logging.basicConfig(level = logging.INFO, format = fmt, handlers = handlers, force = True)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base-dir", default = "/mnt/disks/unslothai/ubuntu/workspace_34/github_scraper")
+    ap.add_argument("--repos", nargs = "+", default = ["unslothai/unsloth", "unslothai/unsloth-zoo"])
+    ap.add_argument("--trial", action = "store_true", help = "Small trial run")
+    ap.add_argument(
+        "--only",
+        nargs = "+",
+        default = None,
+        help = "Only run these resource keys: issues,pulls,discussions,commits,releases,labels,milestones,meta",
+    )
+    ap.add_argument(
+        "--hf-upload-interval",
+        type = int,
+        default = 900,
+        help = "Seconds between HF uploads (0 to disable)",
+    )
+    args = ap.parse_args()
+
+    base = Path(args.base_dir)
+    data_dir = base / "data"
+    data_dir.mkdir(parents = True, exist_ok = True)
+    setup_logging(base / "logs" / f"scraper_{time.strftime('%Y%m%d_%H%M%S')}.log")
+    log.info("Scraper starting: repos=%s trial=%s", args.repos, args.trial)
+
+    client = GitHubClient(min_remaining_graphql = 80, min_remaining_rest = 80)
+    rl = client.rate_snapshot()
+    log.info(
+        "Rate limit snapshot: %s",
+        json.dumps(rl.get("resources", {}), default = str)[:400],
+    )
+
+    # Start HF uploader in background if requested
+    uploader = None
+    if args.hf_upload_interval > 0:
+        from hf_uploader import HFUploader
+        uploader = HFUploader(data_dir, interval_s = args.hf_upload_interval)
+        uploader.start()
+
+    trial_limits = None
+    if args.trial:
+        trial_limits = {
+            "issues": 5,
+            "pull_requests": 5,
+            "discussions": 3,
+            "commits": 20,
+            "releases": 3,
+            "labels": 20,
+            "milestones": 20,
+        }
+
+    only = set(args.only or [])
+
+    try:
+        for repo_spec in args.repos:
+            owner, name = repo_spec.split("/")
+            scraper = RepoScraper(owner, name, data_dir, client, trial_limits)
+            try:
+                repo_meta: Dict[str, Any] = {}
+                if not only or "meta" in only or "commits" in only:
+                    repo_meta = scraper.scrape_repo_meta()
+                if not only or "labels" in only:
+                    scraper.scrape_labels()
+                if not only or "milestones" in only:
+                    scraper.scrape_milestones()
+                if not only or "releases" in only:
+                    scraper.scrape_releases()
+                if not only or "discussions" in only:
+                    scraper.scrape_discussions()
+                if not only or "issues" in only:
+                    scraper.scrape_issues()
+                if not only or "pulls" in only:
+                    scraper.scrape_prs()
+                if not only or "commits" in only:
+                    default_ref = repo_meta.get("defaultBranchRef") or {}
+                    default_branch = (
+                        default_ref.get("name") if isinstance(default_ref, dict) else None
+                    )
+                    branch = f"refs/heads/{default_branch}" if default_branch else "refs/heads/main"
+                    scraper.scrape_commits(branch = branch)
+            finally:
+                scraper.close()
+    finally:
+        if uploader:
+            log.info("Stopping uploader and final sync...")
+            uploader.stop(final_upload = True)
+    log.info(
+        "Scraper complete. GraphQL calls=%d REST calls=%d",
+        client.calls_graphql,
+        client.calls_rest,
+    )
+
+
+if __name__ == "__main__":
+    main()

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py

@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Checkpoint state management for resumable scraping."""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+from pathlib import Path
+from typing import Any, Dict
+
+
+class StateStore:
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents = True, exist_ok = True)
+        self._lock = threading.Lock()
+        self._data: Dict[str, Any] = {}
+        if self.path.exists():
+            try:
+                with self.path.open() as f:
+                    self._data = json.load(f)
+            except Exception:
+                self._data = {}
+
+    def get(
+        self,
+        key: str,
+        default: Any = None,
+    ) -> Any:
+        with self._lock:
+            return self._data.get(key, default)
+
+    def set(self, key: str, value: Any) -> None:
+        with self._lock:
+            self._data[key] = value
+            self._flush()
+
+    def update(self, key: str, **kwargs) -> None:
+        with self._lock:
+            sub = dict(self._data.get(key, {}))
+            sub.update(kwargs)
+            self._data[key] = sub
+            self._flush()
+
+    def all(self) -> Dict[str, Any]:
+        with self._lock:
+            return dict(self._data)
+
+    def _flush(self) -> None:
+        tmp = self.path.with_suffix(self.path.suffix + ".tmp")
+        with tmp.open("w") as f:
+            json.dump(self._data, f, indent = 2, default = str)
+        os.replace(tmp, self.path)
+
+
+class JsonlWriter:
+    """Append-only JSONL writer, thread-safe, with line buffering."""
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents = True, exist_ok = True)
+        self._lock = threading.Lock()
+        self._fh = self.path.open("a", buffering = 1)
+        self._count_seen_keys: set[str] = set()
+        # Preload seen keys for dedup across resumes
+        if self.path.exists() and self.path.stat().st_size > 0:
+            try:
+                with self.path.open() as f:
+                    for line in f:
+                        try:
+                            obj = json.loads(line)
+                            k = self._key(obj)
+                            if k is not None:
+                                self._count_seen_keys.add(k)
+                        except Exception:
+                            pass
+            except Exception:
+                pass
+
+    def _key(self, obj: dict) -> str | None:
+        for k in ("id", "node_id", "number", "sha", "url"):
+            if k in obj:
+                return f"{k}:{obj[k]}"
+        return None
+
+    def has(self, key: str) -> bool:
+        return key in self._count_seen_keys
+
+    def write(self, obj: dict) -> bool:
+        """Return True if newly written, False if already present."""
+        k = self._key(obj)
+        with self._lock:
+            if k is not None and k in self._count_seen_keys:
+                return False
+            if k is not None:
+                self._count_seen_keys.add(k)
+            self._fh.write(json.dumps(obj, default = str, ensure_ascii = False))
+            self._fh.write("\n")
+            self._fh.flush()
+        return True
+
+    def close(self) -> None:
+        try:
+            self._fh.close()
+        except Exception:
+            pass

studio/backend/plugins/data-designer-unstructured-seed/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml

@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "data-designer-unstructured-seed"
+version = "0.1.0"
+description = "Local Data Designer unstructured seed reader plugin"
+requires-python = ">=3.11"
+dependencies = [
+  "data-designer-engine>=0.5.4,<0.6",
+  "pandas>=2,<3",
+  "pymupdf>=1.24.0",
+  "pymupdf4llm>=0.0.17",
+  "mammoth>=1.8.0",
+]
+
+[project.entry-points."data_designer.plugins"]
+unstructured = "data_designer_unstructured_seed.plugin:unstructured_seed_plugin"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/__init__.py

@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from .chunking import (
+    DEFAULT_CHUNK_OVERLAP,
+    DEFAULT_CHUNK_SIZE,
+    build_unstructured_preview_rows,
+    materialize_unstructured_seed_dataset,
+    resolve_chunking,
+)
+from .config import UnstructuredSeedSource
+from .impl import UnstructuredSeedReader
+from .plugin import unstructured_seed_plugin
+
+__all__ = [
+    "DEFAULT_CHUNK_OVERLAP",
+    "DEFAULT_CHUNK_SIZE",
+    "build_unstructured_preview_rows",
+    "materialize_unstructured_seed_dataset",
+    "resolve_chunking",
+    "UnstructuredSeedSource",
+    "UnstructuredSeedReader",
+    "unstructured_seed_plugin",
+]

studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py

@@ -0,0 +1,272 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import hashlib
+import re
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from utils.paths import ensure_dir, unstructured_seed_cache_root
+
+DEFAULT_CHUNK_SIZE = 1200
+DEFAULT_CHUNK_OVERLAP = 200
+MAX_CHUNK_SIZE = 20000
+_MIN_BREAK_RATIO = 0.6
+_CACHE_DIR = unstructured_seed_cache_root()
+
+
+def resolve_chunking(chunk_size: Any, chunk_overlap: Any) -> tuple[int, int]:
+    size = _to_int(chunk_size, DEFAULT_CHUNK_SIZE)
+    size = max(1, min(size, MAX_CHUNK_SIZE))
+    overlap = _to_int(chunk_overlap, DEFAULT_CHUNK_OVERLAP)
+    overlap = max(0, min(overlap, max(0, size - 1)))
+    return size, overlap
+
+
+def build_unstructured_preview_rows(
+    *, source_path: Path, preview_size: int, chunk_size: Any, chunk_overlap: Any
+) -> list[dict[str, str]]:
+    parquet_path, rows = materialize_unstructured_seed_dataset(
+        source_path = source_path,
+        chunk_size = chunk_size,
+        chunk_overlap = chunk_overlap,
+    )
+    count = max(0, int(preview_size))
+    if rows:
+        return rows[:count]
+
+    try:
+        import pandas as pd
+    except ImportError as exc:  # pragma: no cover
+        raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc
+
+    dataframe = pd.read_parquet(parquet_path).head(count)
+    return [
+        {"chunk_text": str(value.get("chunk_text", "")).strip()}
+        for value in dataframe.to_dict(orient = "records")
+        if str(value.get("chunk_text", "")).strip()
+    ]
+
+
+def build_multi_file_preview_rows(
+    *,
+    file_entries: list[tuple[Path, str]],
+    preview_size: int,
+    chunk_size: int | None,
+    chunk_overlap: int | None,
+) -> list[dict[str, str]]:
+    cs = _to_int(chunk_size, DEFAULT_CHUNK_SIZE)
+    co = _to_int(chunk_overlap, DEFAULT_CHUNK_OVERLAP)
+    _, rows = materialize_multi_file_unstructured_seed(
+        file_entries = file_entries,
+        chunk_size = cs,
+        chunk_overlap = co,
+    )
+    return _round_robin_preview(rows, preview_size)
+
+
+def _round_robin_preview(rows: list[dict[str, str]], preview_size: int) -> list[dict[str, str]]:
+    """Pick preview rows round-robin across source files so each is represented."""
+    if not rows or preview_size <= 0:
+        return []
+
+    # Group by source_file, preserving first-appearance order.
+    from collections import OrderedDict
+
+    grouped: OrderedDict[str, list[dict[str, str]]] = OrderedDict()
+    for row in rows:
+        key = row.get("source_file", "")
+        if key not in grouped:
+            grouped[key] = []
+        grouped[key].append(row)
+
+    result: list[dict[str, str]] = []
+    iterators = [iter(chunks) for chunks in grouped.values()]
+    while len(result) < preview_size and iterators:
+        exhausted: list[int] = []
+        for i, it in enumerate(iterators):
+            if len(result) >= preview_size:
+                break
+            val = next(it, None)
+            if val is not None:
+                result.append(val)
+            else:
+                exhausted.append(i)
+        for i in reversed(exhausted):
+            iterators.pop(i)
+
+    return result
+
+
+def materialize_unstructured_seed_dataset(
+    *, source_path: Path, chunk_size: Any, chunk_overlap: Any
+) -> tuple[Path, list[dict[str, str]]]:
+    resolved = source_path.expanduser().resolve()
+    if not resolved.is_file():
+        raise FileNotFoundError(f"unstructured seed file not found: {resolved}")
+
+    size, overlap = resolve_chunking(chunk_size, chunk_overlap)
+    key = _compute_cache_key(
+        source_path = resolved,
+        chunk_size = size,
+        chunk_overlap = overlap,
+    )
+    parquet_path = _CACHE_DIR / f"{key}.parquet"
+    if parquet_path.exists():
+        return parquet_path, []
+
+    text = load_unstructured_text_file(resolved)
+    chunks = split_text_into_chunks(
+        text = text,
+        chunk_size = size,
+        chunk_overlap = overlap,
+    )
+    if not chunks:
+        raise ValueError("No text found in unstructured seed source.")
+
+    rows = [{"chunk_text": chunk} for chunk in chunks]
+    ensure_dir(_CACHE_DIR)
+    try:
+        import pandas as pd
+    except ImportError as exc:  # pragma: no cover
+        raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc
+
+    tmp_path = _CACHE_DIR / f"{key}.tmp.parquet"
+    pd.DataFrame(rows).to_parquet(tmp_path, index = False)
+    tmp_path.replace(parquet_path)
+    return parquet_path, rows
+
+
+def materialize_multi_file_unstructured_seed(
+    *,
+    file_entries: list[tuple[Path, str]],  # (extracted_txt_path, original_filename)
+    chunk_size: int,
+    chunk_overlap: int,
+) -> tuple[Path, list[dict[str, str]]]:
+    """Chunk multiple files into one parquet dataset with a source_file column."""
+    chunk_size, chunk_overlap = resolve_chunking(chunk_size, chunk_overlap)
+    cache_key = _compute_multi_file_cache_key(file_entries, chunk_size, chunk_overlap)
+    cached = _CACHE_DIR / f"{cache_key}.parquet"
+    if cached.exists():
+        df = pd.read_parquet(cached)
+        rows = df.to_dict(orient = "records")
+        return cached, rows
+
+    all_rows: list[dict[str, str]] = []
+    for txt_path, orig_name in file_entries:
+        text = load_unstructured_text_file(txt_path)
+        chunks = split_text_into_chunks(
+            text = text,
+            chunk_size = chunk_size,
+            chunk_overlap = chunk_overlap,
+        )
+        for chunk in chunks:
+            all_rows.append({"chunk_text": chunk, "source_file": orig_name})
+
+    if not all_rows:
+        raise ValueError("No text found in any uploaded files.")
+
+    df = pd.DataFrame(all_rows)
+    ensure_dir(_CACHE_DIR)
+    tmp = _CACHE_DIR / f"{cache_key}.tmp.parquet"
+    df.to_parquet(tmp, index = False)
+    tmp.replace(cached)
+    return cached, all_rows
+
+
+def load_unstructured_text_file(path: Path) -> str:
+    ext = path.suffix.lower()
+    if ext not in {".txt", ".md"}:
+        raise ValueError(f"Unsupported unstructured seed file type: {ext}")
+
+    raw = path.read_text(encoding = "utf-8", errors = "ignore")
+    return normalize_unstructured_text(raw)
+
+
+def normalize_unstructured_text(text: str) -> str:
+    normalized = text.replace("\r\n", "\n").replace("\r", "\n")
+    return re.sub(r"\n{3,}", "\n\n", normalized).strip()
+
+
+def split_text_into_chunks(*, text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
+    if not text:
+        return []
+    if chunk_size <= 0:
+        return [text]
+
+    chunks: list[str] = []
+    start = 0
+    min_break_index = int(chunk_size * _MIN_BREAK_RATIO)
+    text_len = len(text)
+    while start < text_len:
+        end = min(text_len, start + chunk_size)
+        if end < text_len:
+            window = text[start:end]
+            cut = _find_break_index(window, min_break_index)
+            if cut is not None and cut > 0:
+                end = start + cut
+
+        if end <= start:
+            end = min(text_len, start + chunk_size)
+
+        chunk = text[start:end].strip()
+        if chunk:
+            chunks.append(chunk)
+        if end >= text_len:
+            break
+
+        next_start = end - chunk_overlap
+        if next_start <= start:
+            next_start = end
+        start = max(0, next_start)
+
+    return chunks
+
+
+def _find_break_index(window: str, min_index: int) -> int | None:
+    breakpoints = ["\n\n", "\n", " "]
+    for token in breakpoints:
+        idx = window.rfind(token)
+        if idx >= min_index:
+            return idx + len(token)
+    return None
+
+
+def _to_int(value: Any, fallback: int) -> int:
+    if isinstance(value, bool):
+        return fallback
+    try:
+        parsed = int(str(value).strip())
+    except (TypeError, ValueError):
+        return fallback
+    return parsed
+
+
+def _compute_cache_key(*, source_path: Path, chunk_size: int, chunk_overlap: int) -> str:
+    stat = source_path.stat()
+    payload = "|".join(
+        [
+            str(source_path),
+            str(stat.st_size),
+            str(stat.st_mtime_ns),
+            str(chunk_size),
+            str(chunk_overlap),
+        ]
+    ).encode("utf-8")
+    return hashlib.sha256(payload).hexdigest()
+
+
+def _compute_multi_file_cache_key(
+    file_entries: list[tuple[Path, str]], chunk_size: int, chunk_overlap: int
+) -> str:
+    parts: list[str] = []
+    for path, name in sorted(file_entries, key = lambda e: e[1]):
+        st = path.stat()
+        parts.append(f"{path}|{st.st_size}|{st.st_mtime_ns}|{name}")
+    parts.append(f"cs={chunk_size}|co={chunk_overlap}")
+    raw = "\n".join(parts)
+    return hashlib.sha256(raw.encode()).hexdigest()

studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+from pydantic import Field, field_validator, model_validator
+
+from data_designer.config.seed_source import SeedSource
+
+from .chunking import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, resolve_chunking
+
+
+class UnstructuredSeedSource(SeedSource):
+    seed_type: Literal["unstructured"] = "unstructured"
+    paths: list[str] = Field(min_length = 1)
+
+    @model_validator(mode = "before")
+    @classmethod
+    def _normalize_legacy_path(cls, data):
+        if isinstance(data, dict) and "paths" not in data and data.get("path"):
+            data = dict(data)
+            data["paths"] = [data["path"]]
+        return data
+
+    chunk_size: int = DEFAULT_CHUNK_SIZE
+    chunk_overlap: int = DEFAULT_CHUNK_OVERLAP
+
+    @field_validator("paths")
+    @classmethod
+    def _validate_paths(cls, v: list[str]) -> list[str]:
+        for p in v:
+            expanded = Path(p).expanduser()
+            if not expanded.is_file():
+                raise ValueError(f"Seed file does not exist: {expanded}")
+        return v
+
+    @field_validator("chunk_size")
+    @classmethod
+    def _resolve_chunk_size(cls, v: int) -> int:
+        cs, _ = resolve_chunking(v, 0)
+        return cs
+
+    @field_validator("chunk_overlap")
+    @classmethod
+    def _resolve_chunk_overlap(cls, v: int, info) -> int:
+        cs = info.data.get("chunk_size", DEFAULT_CHUNK_SIZE)
+        _, co = resolve_chunking(cs, v)
+        return co

studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py

@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import data_designer.lazy_heavy_imports as lazy
+from data_designer.engine.resources.seed_reader import SeedReader
+
+from .config import UnstructuredSeedSource
+
+
+class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]):
+    def create_duckdb_connection(self):
+        return lazy.duckdb.connect()
+
+    def get_dataset_uri(self) -> str:
+        from .chunking import materialize_multi_file_unstructured_seed
+        import json as json_mod
+
+        file_entries: list[tuple[Path, str]] = []
+        for p in self.source.paths:
+            path_obj = Path(p)
+            file_id = path_obj.name.replace(".extracted.txt", "")
+            meta_path = path_obj.parent / f"{file_id}.meta.json"
+            orig_name = path_obj.name
+            if meta_path.exists():
+                try:
+                    meta = json_mod.loads(meta_path.read_text())
+                    orig_name = meta.get("original_filename", path_obj.name)
+                except (json_mod.JSONDecodeError, OSError):
+                    pass
+            file_entries.append((path_obj, orig_name))
+
+        path, _ = materialize_multi_file_unstructured_seed(
+            file_entries = file_entries,
+            chunk_size = self.source.chunk_size,
+            chunk_overlap = self.source.chunk_overlap,
+        )
+        return str(path)

studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/plugin.py

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from data_designer.plugins.plugin import Plugin, PluginType
+
+unstructured_seed_plugin = Plugin(
+    impl_qualified_name = "data_designer_unstructured_seed.impl.UnstructuredSeedReader",
+    config_qualified_name = "data_designer_unstructured_seed.config.UnstructuredSeedSource",
+    plugin_type = PluginType.SEED_READER,
+)