comet-ml--opik/sdks/opik_optimizer/scripts/datasets/build_bm25_wikipedia.py

#!/usr/bin/env python3
"""
Build and upload BM25 indices for Wikipedia corpus.

This script provides utilities for:
1. Building BM25 search indices from Wikipedia corpus (standard or optimized Parquet format)
2. Optimizing existing indices with Parquet compression
3. Uploading indices to HuggingFace Hub

Usage:
    # Build standard index (JSONL format, ~4.87 GB)
    python scripts/datasets/build_bm25_wikipedia.py build \\
        --output-dir wiki17_abstracts

    # Build optimized index (Parquet format, ~2.5-3.5 GB, 40-50% smaller)
    python scripts/datasets/build_bm25_wikipedia.py build \\
        --output-dir wiki17_abstracts_optimized \\
        --optimize

    # Optimize existing index
    python scripts/datasets/build_bm25_wikipedia.py optimize \\
        --index-dir wiki17_abstracts \\
        --output-dir wiki17_abstracts_optimized

    # Upload index to HuggingFace
    python scripts/datasets/build_bm25_wikipedia.py upload \\
        --index-dir wiki17_abstracts \\
        --repo-id Comet/wikipedia-2017-bm25
"""

import argparse
import logging
import tarfile
import urllib.request
from pathlib import Path

# Setup logging
logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Check for optional dependencies
try:
    import bm25s

    BM25S_AVAILABLE = True
except ImportError:
    BM25S_AVAILABLE = False

try:
    import Stemmer

    STEMMER_AVAILABLE = True
except ImportError:
    STEMMER_AVAILABLE = False


def build_bm25_index(
    output_dir: str = "wiki17_abstracts",
    corpus_jsonl_path: str | None = None,
    download_url: str = "https://huggingface.co/dspy/cache/resolve/main/wiki.abstracts.2017.tar.gz",
    k1: float = 0.9,
    b: float = 0.4,
    optimize: bool = False,
    chunk_size: int = 100000,
) -> None:
    """
    Build BM25 index from Wikipedia 2017 abstracts.

    This downloads the Wikipedia abstracts corpus and builds a bm25s index
    for fair comparison with GEPA paper benchmarks.

    Args:
        output_dir: Directory to save the index
        corpus_jsonl_path: Path to pre-downloaded corpus JSONL file (skips download if provided)
        download_url: URL to download Wikipedia corpus from
        k1: BM25 k1 parameter (term frequency saturation, default 0.9)
        b: BM25 b parameter (length normalization, default 0.4)
        optimize: If True, convert corpus to Parquet format (40-50% smaller)
        chunk_size: Documents per Parquet partition (only used if optimize=True)

    Example:
        # Build standard index (JSONL format, ~4.87 GB)
        build_bm25_index(output_dir="wiki17_abstracts")

        # Build optimized index (Parquet format, ~2.5-3.5 GB)
        build_bm25_index(output_dir="wiki17_abstracts_optimized", optimize=True)

        # Or use existing corpus file
        build_bm25_index(
            output_dir="wiki17_abstracts",
            corpus_jsonl_path="wiki.abstracts.2017.jsonl"
        )

    Note:
        Requires: pip install bm25s[full] PyStemmer
        For optimization: pip install pyarrow
        The corpus is ~3GB compressed, ~8GB uncompressed.
    """
    if not BM25S_AVAILABLE:
        raise ImportError("bm25s not installed. Install with: pip install bm25s[full]")

    if not STEMMER_AVAILABLE:
        raise ImportError(
            "PyStemmer not installed. Install with: pip install PyStemmer"
        )

    try:
        import ujson as json_lib
    except ImportError:
        import json as json_lib  # type: ignore[no-redef]

        logger.warning("ujson not installed, using standard json (slower)")

    output_path = Path(output_dir)
    output_path.mkdir(parents=True, exist_ok=True)

    # Determine corpus file path
    if corpus_jsonl_path:
        jsonl_path = Path(corpus_jsonl_path)
        if not jsonl_path.exists():
            raise FileNotFoundError(f"Corpus file not found: {corpus_jsonl_path}")
    else:
        # Download and extract
        tar_path = output_path / "wiki.abstracts.2017.tar.gz"
        jsonl_path = output_path / "wiki.abstracts.2017.jsonl"

        if not jsonl_path.exists():
            logger.info(f"Downloading Wikipedia corpus from {download_url}")
            logger.info("This is ~3GB and may take several minutes...")

            urllib.request.urlretrieve(download_url, tar_path)

            logger.info(f"Extracting corpus to {output_path}")
            with tarfile.open(tar_path, "r:gz") as tar:
                tar.extractall(path=output_path)

            logger.info("Download and extraction complete")
        else:
            logger.info(f"Using existing corpus at {jsonl_path}")

    # Load corpus
    logger.info("Loading corpus from JSONL...")
    corpus = []
    with open(jsonl_path) as f:
        for line in f:
            entry = json_lib.loads(line)
            # Format: "Title | Text"
            corpus.append(f"{entry['title']} | {' '.join(entry['text'])}")

    logger.info(f"Loaded {len(corpus)} documents")

    # Tokenize corpus
    logger.info("Tokenizing corpus with stemming...")
    stemmer = Stemmer.Stemmer("english")
    corpus_tokens = bm25s.tokenize(corpus, stopwords="en", stemmer=stemmer)

    # Build index
    logger.info(f"Building BM25 index (k1={k1}, b={b})...")
    retriever = bm25s.BM25(k1=k1, b=b)
    retriever.index(corpus_tokens)

    # Save index and corpus
    logger.info(f"Saving index to {output_dir}")

    if optimize:
        # Build with optimized Parquet corpus format
        logger.info("Building with Parquet optimization...")

        # Check for pyarrow
        try:
            import pyarrow as pa
            import pyarrow.parquet as pq
        except ImportError:
            raise ImportError(
                "pyarrow not installed. Install with: pip install pyarrow"
            )

        # Save BM25 index without corpus first
        import tempfile

        with tempfile.TemporaryDirectory() as temp_dir:
            retriever.save(temp_dir, corpus=None)

            # Copy BM25 index files (.npz) to output
            import shutil

            output_path = Path(output_dir)
            output_path.mkdir(parents=True, exist_ok=True)

            temp_path = Path(temp_dir)
            # Copy all BM25 index files (.npy, .npz, .json)
            for pattern in ["*.npz", "*.npy", "*.json"]:
                for index_file in temp_path.glob(pattern):
                    shutil.copy2(index_file, output_path / index_file.name)
                    logger.info(f"Saved {index_file.name}")

        # Convert corpus to chunked Parquet format
        logger.info(f"Converting corpus to Parquet format (chunk_size={chunk_size})...")
        documents = []
        chunk_num = 0

        for i, doc_text in enumerate(corpus):
            # Parse "Title | Text" format
            parts = doc_text.split(" | ", 1)
            title = parts[0].strip() if len(parts) > 0 else ""
            text = parts[1].strip() if len(parts) > 1 else doc_text

            documents.append({"id": i, "title": title, "text": text})

            # Write chunk when reaching chunk_size
            if len(documents) >= chunk_size:
                table = pa.Table.from_pylist(documents)
                pq.write_table(
                    table,
                    output_path / f"corpus_{chunk_num:04d}.parquet",
                    compression="zstd",
                    compression_level=9,
                )
                logger.info(f"Written chunk {chunk_num} ({len(documents)} documents)")
                documents = []
                chunk_num += 1

        # Write remaining documents
        if documents:
            table = pa.Table.from_pylist(documents)
            pq.write_table(
                table,
                output_path / f"corpus_{chunk_num:04d}.parquet",
                compression="zstd",
                compression_level=9,
            )
            logger.info(f"Written final chunk {chunk_num} ({len(documents)} documents)")
    else:
        # Standard JSONL format
        retriever.save(
            output_dir, corpus=corpus, corpus_name="wiki17_abstracts_corpus.jsonl"
        )

    # Calculate and display sizes
    index_size = sum(
        f.stat().st_size for f in Path(output_dir).glob("*") if f.is_file()
    )
    index_size_gb = index_size / (1024**3)

    logger.info(f"\n✅ BM25 index built successfully at {output_dir}")
    logger.info(f"   - Corpus size: {len(corpus)} documents")
    logger.info(f"   - Index size: {index_size_gb:.2f} GB")
    logger.info(
        f"   - Format: {'Parquet (optimized)' if optimize else 'JSONL (standard)'}"
    )
    logger.info(f"   - Parameters: k1={k1}, b={b}")

    if optimize:
        logger.info("\n📦 Optimization enabled:")
        logger.info("   - Corpus format: Chunked Parquet with ZSTD compression")
        logger.info(f"   - Chunk size: {chunk_size} documents")
        logger.info(
            "   - Benefits: 40-50% smaller, streaming access, faster HF downloads"
        )

    logger.info("\n💡 Usage:")
    logger.info("   from opik_optimizer.utils.tools.wikipedia import search_wikipedia")
    logger.info(
        f'   results = search_wikipedia("your query", search_type="bm25", k=5, bm25_index_dir="{output_dir}")'
    )
    logger.info("\n📤 To upload to HuggingFace:")
    logger.info("   python scripts/datasets/build_bm25_wikipedia.py upload \\")
    logger.info(f'       --index-dir "{output_dir}" \\')
    logger.info('       --repo-id "your-org/wikipedia-2017-bm25"')


def optimize_bm25_index(
    input_dir: str,
    output_dir: str,
    chunk_size: int = 100000,
    limit: int | None = None,
) -> None:
    """
    Convert existing BM25 index to optimized Parquet format.

    Args:
        input_dir: Directory with existing BM25 index (JSONL format)
        output_dir: Directory to save optimized index (Parquet format)
        chunk_size: Number of documents per Parquet partition
        limit: Limit number of documents (for testing, default: None = all)

    Example:
        # Optimize existing index
        optimize_bm25_index(
            input_dir="wiki17_abstracts",
            output_dir="wiki17_abstracts_optimized"
        )

        # Test with 1000 documents
        optimize_bm25_index(
            input_dir="wiki17_abstracts",
            output_dir="wiki17_test",
            limit=1000
        )

    Note:
        Requires: pip install pyarrow
        This reduces index size by 40-50% through Parquet + ZSTD compression.
    """
    try:
        import pyarrow as pa
        import pyarrow.parquet as pq
    except ImportError:
        raise ImportError("PyArrow not installed. Install with: pip install pyarrow")

    try:
        import ujson as json_lib
    except ImportError:
        import json as json_lib  # type: ignore[no-redef]

        logger.warning("ujson not installed, using standard json (slower)")

    input_path = Path(input_dir)
    output_path = Path(output_dir)
    output_path.mkdir(parents=True, exist_ok=True)

    # Find the corpus file
    corpus_files = list(input_path.glob("*corpus*.jsonl"))
    if not corpus_files:
        raise FileNotFoundError(f"No corpus JSONL file found in {input_dir}")

    corpus_file = corpus_files[0]
    logger.info(f"Converting {corpus_file.name} to Parquet format...")

    # Read and convert in chunks
    documents = []
    chunk_num = 0

    with open(corpus_file) as f:
        for i, line in enumerate(f):
            # Check limit
            if limit is not None and i >= limit:
                logger.info(f"Reached limit of {limit} documents")
                break

            doc = json_lib.loads(line)

            # bm25s saves corpus as {"id": n, "text": "Title | Text"}
            # Parse the text field to extract title and text
            if isinstance(doc, dict):
                full_text = doc.get("text", "")
                if isinstance(full_text, list):
                    full_text = " ".join(full_text)

                # Split "Title | Text" format
                parts = full_text.split(" | ", 1)
                title = parts[0].strip() if len(parts) > 0 else ""
                text_content = parts[1].strip() if len(parts) > 1 else full_text
            else:
                # Fallback for unexpected format
                parts = str(doc).split(" | ", 1)
                title = parts[0].strip() if len(parts) > 0 else ""
                text_content = parts[1].strip() if len(parts) > 1 else str(doc)

            documents.append({"id": i, "title": title, "text": text_content})

            # Write chunk when reaching chunk_size
            if len(documents) >= chunk_size:
                table = pa.Table.from_pylist(documents)
                pq.write_table(
                    table,
                    output_path / f"corpus_{chunk_num:04d}.parquet",
                    compression="zstd",
                    compression_level=9,
                )
                logger.info(f"Written chunk {chunk_num} ({len(documents)} documents)")
                documents = []
                chunk_num += 1

        # Write remaining documents
        if documents:
            table = pa.Table.from_pylist(documents)
            pq.write_table(
                table,
                output_path / f"corpus_{chunk_num:04d}.parquet",
                compression="zstd",
                compression_level=9,
            )
            logger.info(f"Written final chunk {chunk_num} ({len(documents)} documents)")

    # Copy BM25 index files (.npy, .npz, .json metadata)
    import shutil

    if limit is not None:
        logger.warning(f"⚠️  WARNING: Using --limit={limit} creates an INVALID index!")
        logger.warning(
            f"   The BM25 matrices reference all 5.2M docs, but corpus only has {limit} docs"
        )
        logger.warning("   This index WILL NOT WORK for actual searches!")
        logger.warning(
            "   Only use --limit for testing upload/download, not for searching"
        )
        logger.warning("   Skipping BM25 index file copy...")
        # Don't copy BM25 files when limit is used
    else:
        for pattern in ["*.npz", "*.npy", "*.json"]:
            for index_file in input_path.glob(pattern):
                # Skip the original corpus JSONL
                if "corpus" in index_file.name and index_file.suffix == ".jsonl":
                    continue
                shutil.copy2(index_file, output_path / index_file.name)
                logger.info(f"Copied {index_file.name}")

    # Calculate sizes
    original_size = sum(f.stat().st_size for f in input_path.glob("*") if f.is_file())
    optimized_size = sum(f.stat().st_size for f in output_path.glob("*") if f.is_file())

    logger.info("\n✅ Optimization complete!")
    logger.info(f"   - Original size: {original_size / (1024**3):.2f} GB")
    logger.info(f"   - Optimized size: {optimized_size / (1024**3):.2f} GB")
    logger.info(f"   - Savings: {(1 - optimized_size / original_size) * 100:.1f}%")
    logger.info("\n📦 Chunked corpus format enables:")
    logger.info("   - Streaming access (load chunks on-demand)")
    logger.info("   - Better compression (zstd level 9)")
    logger.info("   - Faster HuggingFace downloads (parallel chunks)")


def upload_bm25_to_huggingface(
    index_dir: str,
    repo_id: str,
    private: bool = False,
) -> str:
    """
    Upload BM25 index to HuggingFace Hub for easy sharing.

    Args:
        index_dir: Local directory containing the bm25s index
        repo_id: HuggingFace repo ID (e.g., "opik-ai/wikipedia-2017-bm25")
        private: Whether to make the repo private (default: False)

    Returns:
        URL of the uploaded repo

    Example:
        # After building the index
        build_bm25_index(output_dir="wiki17_abstracts")

        # Upload to HuggingFace
        url = upload_bm25_to_huggingface(
            index_dir="wiki17_abstracts",
            repo_id="opik-ai/wikipedia-2017-bm25"
        )

        # Now anyone can use it with:
        # from opik_optimizer.utils.tools.wikipedia import get_bm25_search_function
        # search_fn = get_bm25_search_function(hf_repo="opik-ai/wikipedia-2017-bm25")

    Note:
        Requires: pip install huggingface-hub
        You must be logged in: huggingface-cli login
    """
    try:
        from huggingface_hub import HfApi, create_repo
    except ImportError:
        raise ImportError(
            "huggingface_hub not installed. Install with: pip install huggingface-hub"
        )

    index_path = Path(index_dir)
    if not index_path.exists():
        raise FileNotFoundError(f"Index directory not found: {index_dir}")

    logger.info(f"Uploading BM25 index to HuggingFace: {repo_id}")

    # Create repo (dataset type for retrieval indices)
    api = HfApi()
    create_repo(
        repo_id=repo_id,
        repo_type="dataset",
        private=private,
        exist_ok=True,
    )

    logger.info("Uploading files...")

    # Upload the entire directory
    api.upload_folder(
        folder_path=str(index_path),
        repo_id=repo_id,
        repo_type="dataset",
    )

    repo_url = f"https://huggingface.co/datasets/{repo_id}"
    logger.info(f"✅ Successfully uploaded to {repo_url}")

    # Detect format (Parquet or JSONL)
    index_path = Path(index_dir)
    has_parquet = len(list(index_path.glob("*.parquet"))) > 0
    has_jsonl = len(list(index_path.glob("*corpus*.jsonl"))) > 0

    # Calculate actual index size
    index_size = sum(f.stat().st_size for f in index_path.glob("*") if f.is_file())
    index_size_gb = index_size / (1024**3)

    # Generate comprehensive README
    format_info = ""
    if has_parquet:
        format_info = """
## Format: Optimized (Parquet)

This is the **optimized version** with 40-50% size reduction:
- **Corpus Format**: Chunked Parquet with ZSTD compression (level 9)
- **Index Format**: NumPy compressed arrays (.npz)
- **Size**: ~2.5-3.5 GB (vs ~4.87 GB standard)

### Benefits:
- **Smaller downloads**: 40-50% reduction vs standard format
- **Streaming access**: Load only needed document chunks
- **Faster HF downloads**: Parallel chunk downloads
- **Better compression**: ZSTD level 9 on columnar format

### When to use:
- Running on Modal or cloud workers (storage costs matter)
- Bandwidth-constrained environments
- High-volume deployments
"""
    elif has_jsonl:
        format_info = """
## Format: Standard (JSONL)

This is the **standard version** (production-ready):
- **Corpus Format**: JSONL (plain text)
- **Index Format**: NumPy compressed arrays (.npz)
- **Size**: ~4.87 GB

### Benefits:
- **Simple**: Works out of the box, no special handling
- **Widely compatible**: Standard JSON format
- **Good performance**: Already compressed (.npz)

### When to use:
- Local development and testing
- Simple deployments
- When size is not a constraint
"""

    readme_content = f"""---
license: cc-by-sa-3.0
task_categories:
- text-retrieval
language:
- en
tags:
- bm25
- wikipedia
- information-retrieval
- research
size_categories:
- 1M<n<10M
---

# Wikipedia 2017 BM25 Search Index

This dataset provides a production-ready BM25 search index over **5.2 million Wikipedia article abstracts** from the 2017 snapshot. Built using the `bm25s` library with English stemming and optimized Parquet compression, it enables fast, offline information retrieval for research and production AI systems. The corpus is identical to the one used in influential AI research papers including DSPy and GEPA, ensuring reproducible benchmarking and fair comparison across studies.

The index uses BM25 (Best Matching 25), a probabilistic ranking function widely recognized as the gold standard for lexical search. With carefully tuned parameters (k1=0.9 for term frequency saturation, b=0.4 for document length normalization), it provides state-of-the-art retrieval performance for factual queries. Each search completes in under 100ms on consumer hardware, making it suitable for real-time applications, RAG (Retrieval-Augmented Generation) pipelines, and agent tool implementations.

We created this index for the [Opik Optimizer](https://github.com/comet-ml/opik) project to enable reproducible prompt optimization experiments and agent benchmarking. By using the same Wikipedia 2017 corpus as established research, we ensure that optimization results are directly comparable to published baselines. The Parquet-compressed format reduces download size by 67% while maintaining full search fidelity, making it practical for cloud deployments and CI/CD pipelines where storage costs and download times matter.

**Size**: {index_size_gb:.2f} GB | **Format**: {"Parquet (67% compressed)" if has_parquet else "JSONL"} | **Documents**: 5.2M

{format_info}

---

## Quick Start

```python
pip install opik_optimizer[bm25]
```

```python
from opik_optimizer.utils.tools.wikipedia import search_wikipedia

results = search_wikipedia(
    "quantum entanglement",
    search_type="bm25",
    n=5,
    bm25_hf_repo="{repo_id}"
)
```

**That's it!** First run downloads the index (~{index_size_gb:.1f} GB), subsequent searches are instant.

---

## Why Use This?

✅ **Reproducible Research** - Same corpus used in DSPy and GEPA papers
✅ **Fast & Offline** - No API rate limits, <100ms query time
✅ **Production Ready** - Powers RAG systems, Q&A benchmarks, agents
✅ **Memory Efficient** - Optimized Parquet format with chunked loading

---

## Use Cases

**Research & Benchmarking**
- HotpotQA multi-hop question answering
- Information retrieval experiments
- RAG pipeline evaluation
- Agent tool development

**Production Applications**
- Offline knowledge base for AI agents
- Research paper search
- Educational tools
- Content recommendation

---

## Index Specifications

| Attribute | Value |
|-----------|-------|
| Documents | 5,233,330 Wikipedia abstracts |
| Source | Wikipedia 2017 dump (DSPy cache) |
| Tokenization | English stemming + stopword removal |
| Algorithm | BM25 (k1=0.9, b=0.4) |
| Library | [`bm25s`](https://github.com/xhluca/bm25s) |
| Memory | ~6-8 GB RAM during search |
| Query Speed | <100ms per search |

---

## Advanced Usage

### Custom Parameters

```python
from opik_optimizer.utils.tools.wikipedia import search_wikipedia

# Get more results
results = search_wikipedia(
    "machine learning",
    search_type="bm25",
    n=20,  # Top 20 results
    bm25_hf_repo="{repo_id}"
)

# Use with local index (no download)
results = search_wikipedia(
    "neural networks",
    search_type="bm25",
    n=10,
    bm25_index_dir="/path/to/downloaded/index"
)
```

---

## License & Attribution

**Dataset License**: [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) (Wikipedia content license)
**Source**: Wikipedia 2017 abstracts from [DSPy cache](https://huggingface.co/dspy/cache)

**Citation**: If you use this in research, please cite:
- This dataset: `{repo_id}`

---

## Related Links and Datasets
- [wikipedia dataset](https://huggingface.co/datasets/wikipedia) Full Wikipedia dumps (all languages)
- [opik_optimizer](https://github.com/comet-ml/opik/tree/main/sdks/opik_optimizer) repository


**Built with [opik_optimizer](https://github.com/comet-ml/opik) by Comet**
Thanks to [@vincentkoc](https://github.com/vincentkoc) from the Comet team for creating this Parquet version.
"""

    try:
        api.upload_file(
            path_or_fileobj=readme_content.encode(),
            path_in_repo="README.md",
            repo_id=repo_id,
            repo_type="dataset",
        )
    except Exception:
        logger.warning("Could not create README (may already exist)")

    return repo_url


def main() -> None:
    """CLI interface for building and uploading BM25 indices."""
    parser = argparse.ArgumentParser(
        description="Build and upload BM25 Wikipedia indices",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__,
    )
    subparsers = parser.add_subparsers(dest="command", required=True)

    # Build command
    build_parser = subparsers.add_parser(
        "build", help="Build BM25 index from Wikipedia corpus"
    )
    build_parser.add_argument(
        "--output-dir",
        default="wiki17_abstracts",
        help="Output directory for index (default: wiki17_abstracts)",
    )
    build_parser.add_argument(
        "--corpus-jsonl",
        help="Path to pre-downloaded corpus JSONL file (skips download)",
    )
    build_parser.add_argument(
        "--download-url",
        default="https://huggingface.co/dspy/cache/resolve/main/wiki.abstracts.2017.tar.gz",
        help="URL to download corpus from",
    )
    build_parser.add_argument(
        "--k1", type=float, default=0.9, help="BM25 k1 parameter (default: 0.9)"
    )
    build_parser.add_argument(
        "--b", type=float, default=0.4, help="BM25 b parameter (default: 0.4)"
    )
    build_parser.add_argument(
        "--optimize",
        action="store_true",
        help="Build with Parquet optimization (40-50%% smaller, requires pyarrow)",
    )
    build_parser.add_argument(
        "--chunk-size",
        type=int,
        default=100000,
        help="Documents per Parquet chunk (default: 100000, only used with --optimize)",
    )

    # Optimize command
    optimize_parser = subparsers.add_parser(
        "optimize", help="Optimize existing index to Parquet format"
    )
    optimize_parser.add_argument(
        "--index-dir",
        required=True,
        help="Directory containing existing BM25 index (JSONL format)",
    )
    optimize_parser.add_argument(
        "--output-dir",
        required=True,
        help="Directory to save optimized index (Parquet format)",
    )
    optimize_parser.add_argument(
        "--chunk-size",
        type=int,
        default=100000,
        help="Documents per Parquet partition (default: 100000)",
    )
    optimize_parser.add_argument(
        "--limit",
        type=int,
        help="Limit number of documents (for testing, e.g., --limit 1000)",
    )

    # Upload command
    upload_parser = subparsers.add_parser(
        "upload", help="Upload index to HuggingFace Hub"
    )
    upload_parser.add_argument(
        "--index-dir", required=True, help="Directory containing the built BM25 index"
    )
    upload_parser.add_argument(
        "--repo-id",
        required=True,
        help="HuggingFace repo ID (e.g., 'opik-ai/wikipedia-2017-bm25')",
    )
    upload_parser.add_argument(
        "--private", action="store_true", help="Make the repository private"
    )

    args = parser.parse_args()

    if args.command == "build":
        build_bm25_index(
            output_dir=args.output_dir,
            corpus_jsonl_path=args.corpus_jsonl,
            download_url=args.download_url,
            k1=args.k1,
            b=args.b,
            optimize=args.optimize,
            chunk_size=args.chunk_size,
        )
    elif args.command == "optimize":
        optimize_bm25_index(
            input_dir=args.index_dir,
            output_dir=args.output_dir,
            chunk_size=args.chunk_size,
            limit=args.limit,
        )
    elif args.command == "upload":
        upload_bm25_to_huggingface(
            index_dir=args.index_dir,
            repo_id=args.repo_id,
            private=args.private,
        )


if __name__ == "__main__":
    main()