chore: import upstream snapshot with attribution

benchmarks/benchmark.py

@@ -0,0 +1,438 @@
+#!/usr/bin/env python3
+# Copyright 2025 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""LangExtract benchmark suite for performance and quality testing.
+
+Measures tokenization speed and extraction quality across multiple languages
+and text types. Automatically downloads test texts from Project Gutenberg
+and generates comparative visualizations.
+
+Usage:
+  # Run diverse text type benchmark (default)
+  python benchmarks/benchmark.py
+
+  # Test with specific model
+  python benchmarks/benchmark.py --model gemini-2.5-flash
+  python benchmarks/benchmark.py --model gemma2:2b  # Local model via Ollama
+
+  # Generate comparison plots from existing results
+  python benchmarks/benchmark.py --compare
+
+Requirements:
+  - Set GEMINI_API_KEY for cloud models
+  - Install Ollama for local model testing
+  - Results saved to benchmark_results/
+"""
+
+import argparse
+from datetime import datetime
+import json
+import os
+from pathlib import Path
+import time
+from typing import Any
+import urllib.error
+
+import dotenv
+
+from benchmarks import config
+from benchmarks import plotting
+from benchmarks import utils
+import langextract
+from langextract import core
+from langextract import data
+from langextract import visualize
+import langextract.io as lio
+
+# Load API key from environment
+dotenv.load_dotenv(override=True)
+GEMINI_API_KEY = os.environ.get(
+    "GEMINI_API_KEY", os.environ.get("LANGEXTRACT_API_KEY")
+)
+
+
+class BenchmarkRunner:
+  """Orchestrates benchmark execution and result collection."""
+
+  def __init__(self):
+    """Initialize runner with timestamp and git metadata."""
+    self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    self.git_info = utils.get_git_info()
+    self.tokenizer = core.tokenizer.RegexTokenizer()
+
+  def set_tokenizer(self, tokenizer_type: str):
+    """Set the tokenizer to use."""
+    if tokenizer_type.lower() == "unicode":
+      self.tokenizer = core.tokenizer.UnicodeTokenizer()
+      print("Using UnicodeTokenizer")
+    else:
+      self.tokenizer = core.tokenizer.RegexTokenizer()
+      print("Using RegexTokenizer (default)")
+
+  def print_header(self):
+    """Print benchmark header."""
+    print("=" * config.DISPLAY.separator_width)
+    print("LANGEXTRACT BENCHMARK")
+    print("=" * config.DISPLAY.separator_width)
+    print(
+        f"Branch: {self.git_info['branch']} | Commit: {self.git_info['commit']}"
+    )
+    print("-" * config.DISPLAY.separator_width)
+
+  def benchmark_tokenization(self) -> list[dict[str, Any]]:
+    """Measure tokenization throughput at different text sizes.
+
+    Returns:
+      List of dicts with words, tokens, timing, and throughput metrics.
+    """
+    print("\nTokenization Performance")
+    print("-" * config.DISPLAY.subseparator_width)
+
+    results = []
+
+    for word_count in config.TOKENIZATION.default_text_sizes:
+      text = " ".join(["word"] * word_count)
+
+      _ = self.tokenizer.tokenize(text)
+
+      times = []
+      for _ in range(config.TOKENIZATION.benchmark_iterations):
+        start = time.perf_counter()
+        tokenized = self.tokenizer.tokenize(text)
+        elapsed = time.perf_counter() - start
+        times.append(elapsed)
+
+      avg_time = sum(times) / len(times)
+      avg_ms = avg_time * 1000
+      num_tokens = len(tokenized.tokens)
+      tokens_per_sec = num_tokens / avg_time if avg_time > 0 else 0
+
+      word_str = (
+          f"{word_count//1000:,}k" if word_count >= 1000 else f"{word_count:,}"
+      )
+
+      print(
+          f"{word_str:>6} words: {avg_ms:7.2f}ms  "
+          f"({tokens_per_sec/1e6:.1f}M tokens/sec)"
+      )
+
+      results.append({
+          "words": word_count,
+          "tokens": num_tokens,
+          "avg_ms": avg_ms,
+          "tokens_per_sec": tokens_per_sec,
+      })
+
+    return results
+
+  def test_single_extraction(
+      self,
+      model_id: str = config.MODELS.default_model,
+      text_type: config.TextTypes = config.TextTypes.ENGLISH,
+  ) -> dict[str, Any]:
+    """Execute extraction test.
+
+    Args:
+      model_id: Model identifier (e.g., 'gemini-2.5-flash', 'gemma2:2b').
+      text_type: Language/text type to test.
+
+    Returns:
+      Dict with success status, timing, entity counts, and metrics.
+    """
+    print("\nExtraction Test")
+    print("-" * config.DISPLAY.subseparator_width)
+
+    try:
+      # Get test text
+      test_text = utils.get_text_from_gutenberg(text_type)
+      test_text = utils.get_optimal_text_size(test_text, model_id)
+
+      print(f"   Text: {len(test_text):,} characters ({text_type.value})")
+      print(f"   Model: {model_id}")
+
+      # Analyze tokenization
+      tokenization_analysis = utils.analyze_tokenization(
+          test_text, self.tokenizer
+      )
+      print(
+          "   Tokenization:"
+          f" {utils.format_tokenization_summary(tokenization_analysis)}"
+      )
+
+      # Get extraction config for text type
+      extraction_config = utils.get_extraction_example(text_type)
+
+      example = data.ExampleData(
+          text="MACBETH speaks to LADY MACBETH about Duncan.",
+          extractions=[
+              data.Extraction(
+                  extraction_text="Macbeth", extraction_class="Character"
+              ),
+              data.Extraction(
+                  extraction_text="Lady Macbeth", extraction_class="Character"
+              ),
+              data.Extraction(
+                  extraction_text="Duncan", extraction_class="Character"
+              ),
+          ],
+      )
+
+      max_retries = 5
+      retry_delay = 3.0
+
+      # Retry logic for transient network/API failures
+      for attempt in range(max_retries):
+        try:
+          start_time = time.time()
+          result = langextract.extract(
+              text_or_documents=test_text,
+              model_id=model_id,
+              api_key=GEMINI_API_KEY,
+              prompt_description=extraction_config["prompt"],
+              examples=[example],
+              max_workers=config.MODELS.default_max_workers,
+              temperature=config.MODELS.default_temperature,
+              extraction_passes=config.MODELS.default_extraction_passes,
+              tokenizer=self.tokenizer,
+          )
+          elapsed = time.time() - start_time
+          break
+        except (ConnectionError, TimeoutError):
+          if attempt < max_retries - 1:
+            print(f"   Retrying in {retry_delay}s...")
+            time.sleep(retry_delay)
+            retry_delay *= 1.5
+            continue
+          raise
+
+      print(f"Extraction completed in {elapsed:.1f}s")
+
+      grounded_entities = []
+      ungrounded_entities = []
+
+      if result.extractions:
+        for extraction in result.extractions:
+          is_grounded = (
+              extraction.char_interval
+              and extraction.char_interval.start_pos is not None
+              and extraction.char_interval.end_pos is not None
+          )
+
+          entity_text = extraction.extraction_text
+          if entity_text:
+            if is_grounded:
+              grounded_entities.append(entity_text)
+            else:
+              ungrounded_entities.append(entity_text)
+
+      unique_grounded = list(set(grounded_entities))
+      unique_ungrounded = list(set(ungrounded_entities))
+
+      print(f"Found {len(unique_grounded)} grounded entities")
+      if unique_ungrounded:
+        print(f"   ({len(unique_ungrounded)} ungrounded entities ignored)")
+
+      if unique_grounded:
+        sample = unique_grounded[:5]
+        sample_str = ", ".join(sample) + (
+            "..." if len(unique_grounded) > 5 else ""
+        )
+        print(f"   Sample: {sample_str}")
+
+      return {
+          "success": True,
+          "model": model_id,
+          "text_type": text_type.value,
+          "time_seconds": elapsed,
+          "entity_count": len(unique_grounded),
+          "ungrounded_count": len(unique_ungrounded),
+          "sample_entities": unique_grounded[:10],
+          "tokenization": tokenization_analysis,
+          config.EXTRACTION_RESULT_KEY: result,
+      }
+
+    except (urllib.error.URLError, RuntimeError) as e:
+      # Handle expected text download failures.
+      print(f"Failed: {e}")
+      return {
+          "success": False,
+          "model": model_id,
+          "text_type": text_type.value,
+          "error": str(e),
+      }
+
+  def test_diverse_text_types(
+      self, models: list[str] | None = None
+  ) -> list[dict[str, Any]]:
+    """Test extraction with diverse text types."""
+    print("\n" + "=" * config.DISPLAY.separator_width)
+    print("DIVERSE TEXT TYPE MODE")
+    print("=" * config.DISPLAY.separator_width)
+
+    if models is None:
+      models = [config.MODELS.default_model]
+
+    results = []
+    test_count = 0
+
+    for model_id in models:
+      print(f"\nTesting {model_id}")
+      print("-" * 30)
+
+      for text_type in config.TextTypes:
+        print(f"\n  Testing {text_type.value} text...")
+        result = self.test_single_extraction(model_id, text_type)
+        results.append(result)
+
+        if result.get("success"):
+          test_count += 1
+          if test_count % 3 == 0:
+            print(
+                "   Rate limit delay"
+                f" ({config.MODELS.gemini_rate_limit_delay}s)..."
+            )
+            time.sleep(config.MODELS.gemini_rate_limit_delay)
+
+    print(f"\nCompleted {test_count} successful tests")
+    return results
+
+  def save_results(self, results: dict[str, Any]):
+    """Save results and create plots."""
+    results["timestamp"] = self.timestamp
+    results["git"] = self.git_info
+
+    json_path = config.PATHS.get_result_path(self.timestamp, "").with_suffix(
+        ".json"
+    )
+
+    viz_dir = json_path.parent / "visualizations" / self.timestamp
+    viz_dir.mkdir(parents=True, exist_ok=True)
+
+    if config.RESULTS_KEY in results:
+      print(f"\nGenerating visualizations in: {viz_dir}")
+      for result in results[config.RESULTS_KEY]:
+        if result.get("success") and config.EXTRACTION_RESULT_KEY in result:
+          model_name = result["model"].replace("/", "_").replace(":", "_")
+          text_type = result["text_type"]
+          viz_name = f"{model_name}_{text_type}"
+
+          jsonl_path = viz_dir / f"{viz_name}.jsonl"
+          lio.save_annotated_documents(
+              [result[config.EXTRACTION_RESULT_KEY]],
+              output_name=jsonl_path.name,
+              output_dir=str(viz_dir),
+          )
+
+          html_content = visualize(str(jsonl_path))
+          html_path = viz_dir / f"{viz_name}.html"
+          with open(html_path, "w") as f:
+            f.write(getattr(html_content, "data", html_content))
+
+    # Remove extraction result objects before saving JSON
+    for result in results.get(config.RESULTS_KEY, []):
+      result.pop(config.EXTRACTION_RESULT_KEY, None)
+
+    with open(json_path, "w") as f:
+      json.dump(results, f, indent=2, default=str)
+    print(f"\nResults saved to: {json_path}")
+
+    plot_created = plotting.create_diverse_plots(results, json_path)
+
+    if plot_created:
+      print(f"Plot saved to: {json_path.with_suffix('.png')}")
+    else:
+      print(f"Warning: Failed to create plot for {json_path.name}")
+
+  def run_diverse_benchmark(self, models: list[str] | None = None):
+    """Run benchmark."""
+    self.print_header()
+
+    tokenization_results = self.benchmark_tokenization()
+    diverse_results = self.test_diverse_text_types(models)
+
+    results = {
+        "tokenization": tokenization_results,
+        config.RESULTS_KEY: diverse_results,
+    }
+
+    self.save_results(results)
+
+
+def main():
+  """Main entry point."""
+  parser = argparse.ArgumentParser(description="LangExtract Benchmark Suite")
+
+  parser.add_argument(
+      "--model",
+      type=str,
+      default=None,
+      help=f"Model to use (default: {config.MODELS.default_model})",
+  )
+
+  parser.add_argument(
+      "--tokenizer",
+      type=str,
+      choices=["regex", "unicode"],
+      default="regex",
+      help="Tokenizer to use (default: regex)",
+  )
+
+  parser.add_argument(
+      "--compare",
+      action="store_true",
+      help="Generate comparison plots from existing benchmark results",
+  )
+
+  args = parser.parse_args()
+
+  # Handle comparison mode
+  if args.compare:
+    results_dir = Path("benchmark_results")
+    json_files = sorted(results_dir.glob("benchmark_*.json"))
+
+    if len(json_files) < 2:
+      print(
+          "Need at least 2 benchmark results for comparison, found"
+          f" {len(json_files)}"
+      )
+      return
+
+    print(f"Found {len(json_files)} benchmark results to compare")
+
+    # Use last 10 results or all if less than 10
+    files_to_compare = json_files[-10:]
+    comparison_path = (
+        results_dir
+        / f"comparison_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
+    )
+
+    plotting.create_comparison_plots(files_to_compare, comparison_path)
+    print(f"\nComparison plot saved to: {comparison_path}")
+    return
+
+  model_to_test = args.model or config.MODELS.default_model
+  if "gemini" in model_to_test.lower() and not GEMINI_API_KEY:
+    print(
+        f"Error: {model_to_test} requires GEMINI_API_KEY or LANGEXTRACT_API_KEY"
+    )
+    return
+
+  runner = BenchmarkRunner()
+  runner.set_tokenizer(args.tokenizer)
+  runner.run_diverse_benchmark([args.model] if args.model else None)
+
+
+if __name__ == "__main__":
+  main()

benchmarks/config.py

@@ -0,0 +1,107 @@
+# Copyright 2025 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Benchmark configuration settings and constants.
+
+Centralized configuration for tokenization tests, model parameters,
+display formatting, and test text sources.
+"""
+
+from dataclasses import dataclass
+import enum
+from pathlib import Path
+
+# Result dictionary keys
+RESULTS_KEY = "results"
+EXTRACTION_KEY = "extraction"
+EXTRACTION_RESULT_KEY = "extraction_result"
+TOKENIZATION_KEY = "tokenization"
+
+
+@dataclass(frozen=True)
+class TokenizationConfig:
+  """Settings for tokenization performance tests."""
+
+  default_text_sizes: tuple[int, ...] = (100, 1000, 10000)  # Word counts
+  benchmark_iterations: int = 10  # Iterations per size for averaging
+
+
+@dataclass(frozen=True)
+class ModelConfig:
+  """Model and API configuration."""
+
+  default_model: str = "gemini-2.5-flash"  # Cloud model default
+  local_model: str = "gemma2:9b"  # Ollama model default
+  default_temperature: float = 0.0  # Deterministic output
+  default_max_workers: int = 10  # Parallel processing threads
+  default_extraction_passes: int = 1  # Single pass extraction
+  gemini_rate_limit_delay: float = 8.0  # Seconds between batches
+
+
+class TextTypes(str, enum.Enum):
+  """Supported languages for extraction testing."""
+
+  ENGLISH = "english"
+  JAPANESE = "japanese"
+  FRENCH = "french"
+  SPANISH = "spanish"
+
+
+# Test texts from Project Gutenberg (similar genres for fair comparison)
+GUTENBERG_TEXTS = {
+    TextTypes.ENGLISH: (
+        "https://www.gutenberg.org/files/11/11-0.txt"
+    ),  # Alice's Adventures
+    TextTypes.JAPANESE: (
+        "https://www.gutenberg.org/files/1982/1982-0.txt"
+    ),  # Rashomon
+    TextTypes.FRENCH: (
+        "https://www.gutenberg.org/files/55456/55456-0.txt"
+    ),  # Alice (French)
+    TextTypes.SPANISH: (
+        "https://www.gutenberg.org/files/67248/67248-0.txt"
+    ),  # El clavo
+}
+
+
+@dataclass(frozen=True)
+class DisplayConfig:
+  """Display configuration."""
+
+  separator_width: int = 50
+  subseparator_width: int = 40
+  figure_size_single: tuple[int, int] = (12, 5)
+  figure_size_multi: tuple[int, int] = (14, 10)
+  plot_style: str = "seaborn-v0_8-darkgrid"
+
+
+@dataclass(frozen=True)
+class PathConfig:
+  """Path configuration."""
+
+  results_dir: Path = Path("benchmark_results")
+
+  def get_result_path(self, timestamp: str, suffix: str = "") -> Path:
+    """Get result file path."""
+    if not self.results_dir.exists():
+      self.results_dir.mkdir(parents=True)
+    filename = f"benchmark{suffix}_{timestamp}"
+    return self.results_dir / filename
+
+
+# Global config instances
+TOKENIZATION = TokenizationConfig()
+MODELS = ModelConfig()
+DISPLAY = DisplayConfig()
+PATHS = PathConfig()

benchmarks/fuzzy_benchmark.py

@@ -0,0 +1,505 @@
+#!/usr/bin/env python3
+# Copyright 2025 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Benchmark for fuzzy alignment in the resolver.
+
+Measures wall-time and correctness of _fuzzy_align_extraction across
+realistic input sizes. Run from repo root:
+
+  python benchmarks/fuzzy_benchmark.py
+  python benchmarks/fuzzy_benchmark.py --sizes planted_contiguous,perf_1k
+  python benchmarks/fuzzy_benchmark.py --sizes large --runs 1
+  python benchmarks/fuzzy_benchmark.py --tokenizer unicode
+  python benchmarks/fuzzy_benchmark.py --algorithm lcs
+  python benchmarks/fuzzy_benchmark.py --algorithm legacy
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import platform
+import random
+import subprocess
+import sys
+import time
+
+from langextract import resolver as resolver_lib
+from langextract.core import data
+from langextract.core import tokenizer as tokenizer_lib
+
+_WORD_POOL = [
+    "patient",
+    "diagnosed",
+    "with",
+    "diabetes",
+    "hypertension",
+    "medication",
+    "prescribed",
+    "daily",
+    "chronic",
+    "condition",
+    "treatment",
+    "history",
+    "symptoms",
+    "blood",
+    "pressure",
+    "glucose",
+    "insulin",
+    "kidney",
+    "liver",
+    "cardiac",
+    "pulmonary",
+    "neurological",
+    "assessment",
+    "examination",
+    "laboratory",
+    "results",
+    "normal",
+    "elevated",
+    "decreased",
+    "follow",
+    "appointment",
+    "scheduled",
+    "monitor",
+    "progress",
+    "clinical",
+    "evaluation",
+    "imaging",
+    "therapy",
+    "dosage",
+    "adverse",
+    "reaction",
+    "prognosis",
+    "referral",
+    "discharge",
+    "admission",
+    "surgery",
+    "recovery",
+    "emergency",
+    "outpatient",
+    "inpatient",
+    "consultation",
+    "diagnosis",
+    "pathology",
+    "specimen",
+    "biopsy",
+    "cultures",
+    "antibiotics",
+    "analgesic",
+    "sedation",
+    "ventilation",
+    "intubation",
+    "catheter",
+    "drainage",
+    "infusion",
+]
+
+
+def _generate_source_text(n_tokens: int, seed: int = 42) -> str:
+  """Generates deterministic source text from _WORD_POOL."""
+  rng = random.Random(seed)
+  words = [rng.choice(_WORD_POOL) for _ in range(n_tokens)]
+  return " ".join(words)
+
+
+def _plant_span(source: str, target: str, position: int) -> str:
+  """Inserts target text at approximately token position in source."""
+  words = source.split()
+  target_words = target.split()
+  pos = min(position, len(words))
+  words[pos : pos + len(target_words)] = target_words
+  return " ".join(words)
+
+
+def _plant_gapped(source: str, tokens: list[str], start: int, gap: int) -> str:
+  """Inserts tokens with gaps between them in source."""
+  words = source.split()
+  for i, token in enumerate(tokens):
+    pos = min(start + i * (gap + 1), len(words) - 1)
+    words[pos] = token
+  return " ".join(words)
+
+
+def _make_extraction(text: str) -> data.Extraction:
+  return data.Extraction(
+      extraction_class="entity",
+      extraction_text=text,
+  )
+
+
+def _build_cases() -> dict[str, dict]:
+  """Builds benchmark cases with planted spans for correctness oracles."""
+  cases = {}
+
+  # --- Planted correctness cases (small, fast) ---
+
+  base_200 = _generate_source_text(200, seed=42)
+
+  # Contiguous positive: plant exact 3-token span at known position.
+  planted_source = _plant_span(base_200, "metformin hydrochloride tablet", 50)
+  cases["planted_contiguous"] = {
+      "description": "3-token planted contiguous match in 200 tokens",
+      "source": planted_source,
+      "extraction_text": "metformin hydrochloride tablet",
+      "expect_match": True,
+      "expect_token_interval": (50, 53),
+      "expect_char_interval": (451, 481),
+      "expect_substring": "metformin hydrochloride tablet",
+  }
+
+  # Fuzzy positive: extraction has stemming variation.
+  cases["planted_fuzzy"] = {
+      "description": "3-token fuzzy match (stemming) in 200 tokens",
+      "source": planted_source,
+      "extraction_text": "metformins hydrochlorides tablets",
+      "expect_match": True,
+      "expect_token_interval": (50, 53),
+      "expect_char_interval": (451, 481),
+      "expect_substring": "metformin hydrochloride tablet",
+  }
+
+  # Gapped positive: extraction tokens scattered with noise between them.
+  gapped_source = _plant_gapped(
+      _generate_source_text(200, seed=99),
+      ["metformin", "hydrochloride", "tablet"],
+      start=40,
+      gap=3,
+  )
+  cases["planted_gapped"] = {
+      "description": "3-token gapped match (gap=3) in 200 tokens",
+      "source": gapped_source,
+      "extraction_text": "metformin hydrochloride tablet",
+      "expect_match": True,
+      "expect_token_interval": (40, 49),
+      "expect_char_interval": (371, 461),
+      "expect_substring": (
+          "metformin pulmonary antibiotics assessment"
+          " hydrochloride hypertension pressure with tablet"
+      ),
+  }
+
+  # Near-miss negative: tokens not present in source.
+  cases["planted_negative"] = {
+      "description": "3-token near-miss negative in 200 tokens",
+      "source": base_200,
+      "extraction_text": "warfarin coumadin anticoagulant",
+      "expect_match": False,
+  }
+
+  # --- Perf stress case (in-vocabulary extraction, keeps overlap filter hot) ---
+
+  source_perf = _generate_source_text(1000, seed=42)
+  cases["perf_1k"] = {
+      "description": "5-token in-vocab extraction, 1000-token source (perf)",
+      "source": source_perf,
+      "extraction_text": "patient diagnosed chronic condition treatment",
+  }
+
+  # --- Scale cases (opt-in) ---
+
+  source_large = _generate_source_text(5000, seed=42)
+  cases["large"] = {
+      "description": "5-token in-vocab extraction, 5000-token source (opt-in)",
+      "source": source_large,
+      "extraction_text": "patient diagnosed chronic condition treatment",
+  }
+
+  source_stress = _generate_source_text(10000, seed=42)
+  cases["stress"] = {
+      "description": "5-token in-vocab extraction, 10000-token source (opt-in)",
+      "source": source_stress,
+      "extraction_text": "patient diagnosed chronic condition treatment",
+  }
+
+  return cases
+
+
+_DEFAULT_SIZES = (
+    "planted_contiguous,planted_fuzzy,planted_gapped,planted_negative,perf_1k"
+)
+
+
+def _get_metadata(
+    tokenizer_name: str,
+    seed: int,
+    threshold: float,
+    algorithm: str,
+    min_density: float,
+) -> dict:
+  """Collects run metadata for reproducibility."""
+  git_sha = "unknown"
+  try:
+    git_sha = (
+        subprocess.check_output(
+            ["git", "rev-parse", "--short", "HEAD"],
+            stderr=subprocess.DEVNULL,
+        )
+        .decode()
+        .strip()
+    )
+  except (subprocess.CalledProcessError, FileNotFoundError):
+    pass
+
+  return {
+      "python_version": platform.python_version(),
+      "platform": platform.platform(),
+      "tokenizer": tokenizer_name,
+      "seed": seed,
+      "fuzzy_alignment_threshold": threshold,
+      "fuzzy_alignment_algorithm": algorithm,
+      "fuzzy_alignment_min_density": min_density,
+      "git_sha": git_sha,
+  }
+
+
+def _run_single(
+    aligner: resolver_lib.WordAligner,
+    source_text: str,
+    extraction_text: str,
+    tokenizer: tokenizer_lib.Tokenizer,
+    threshold: float,
+    algorithm: str,
+    min_density: float,
+) -> dict:
+  """Runs a single fuzzy alignment and returns timing + result."""
+  resolver_lib._normalize_token.cache_clear()
+
+  tokenized = tokenizer.tokenize(source_text)
+  source_tokens = [t.lower() for t in _tokenize_words(source_text, tokenizer)]
+  extraction = _make_extraction(extraction_text)
+
+  start = time.perf_counter()
+  if algorithm == "lcs":
+    result = aligner._lcs_fuzzy_align_extraction(
+        extraction=extraction,
+        source_tokens=source_tokens,
+        tokenized_text=tokenized,
+        token_offset=0,
+        char_offset=0,
+        fuzzy_alignment_threshold=threshold,
+        fuzzy_alignment_min_density=min_density,
+        tokenizer_impl=tokenizer,
+    )
+  else:
+    result = aligner._fuzzy_align_extraction(
+        extraction=extraction,
+        source_tokens=source_tokens,
+        tokenized_text=tokenized,
+        token_offset=0,
+        char_offset=0,
+        fuzzy_alignment_threshold=threshold,
+        tokenizer_impl=tokenizer,
+    )
+  elapsed = time.perf_counter() - start
+
+  matched_substring = None
+  if result and result.char_interval:
+    start_pos = result.char_interval.start_pos
+    end_pos = result.char_interval.end_pos
+    matched_substring = source_text[start_pos:end_pos]
+
+  return {
+      "elapsed_ms": round(elapsed * 1000, 2),
+      "matched": result is not None,
+      "alignment_status": result.alignment_status.value if result else None,
+      "token_interval": (
+          f"{result.token_interval.start_index}"
+          f"-{result.token_interval.end_index}"
+          if result and result.token_interval
+          else None
+      ),
+      "char_interval": (
+          f"{result.char_interval.start_pos}-{result.char_interval.end_pos}"
+          if result and result.char_interval
+          else None
+      ),
+      "matched_substring": matched_substring,
+  }
+
+
+def _tokenize_words(text: str, tokenizer: tokenizer_lib.Tokenizer) -> list[str]:
+  """Extracts word strings from tokenized text."""
+  tokenized = tokenizer.tokenize(text)
+  return [
+      text[t.char_interval.start_pos : t.char_interval.end_pos]
+      for t in tokenized.tokens
+  ]
+
+
+def main():
+  parser = argparse.ArgumentParser(
+      description="Benchmark fuzzy alignment performance"
+  )
+  parser.add_argument(
+      "--sizes",
+      default=_DEFAULT_SIZES,
+      help="Comma-separated case names (default: planted + perf_1k)",
+  )
+  parser.add_argument(
+      "--runs", type=int, default=3, help="Number of runs per case"
+  )
+  parser.add_argument(
+      "--tokenizer",
+      choices=["regex", "unicode"],
+      default="regex",
+      help="Tokenizer backend (default: regex)",
+  )
+  parser.add_argument(
+      "--threshold",
+      type=float,
+      default=0.75,
+      help="Fuzzy alignment threshold (default: 0.75)",
+  )
+  parser.add_argument(
+      "--algorithm",
+      choices=["lcs", "legacy"],
+      default="lcs",
+      help="Fuzzy alignment algorithm (default: lcs)",
+  )
+  parser.add_argument(
+      "--min-density",
+      type=float,
+      default=1 / 3,
+      help="Min matched-to-span density for LCS algorithm (default: 1/3)",
+  )
+  parser.add_argument("--json-output", help="Write results to JSON file")
+  args = parser.parse_args()
+
+  cases = _build_cases()
+  selected = [s.strip() for s in args.sizes.split(",")]
+
+  if args.tokenizer == "unicode":
+    tokenizer = tokenizer_lib.UnicodeTokenizer()
+  else:
+    tokenizer = tokenizer_lib.RegexTokenizer()
+
+  aligner = resolver_lib.WordAligner()
+  metadata = _get_metadata(
+      args.tokenizer, 42, args.threshold, args.algorithm, args.min_density
+  )
+
+  results = {"_metadata": metadata}
+  print(f"Fuzzy alignment benchmark ({args.runs} runs per case)\n")
+  print(f"  algorithm: {args.algorithm}")
+  print(f"  tokenizer: {args.tokenizer}")
+  print(f"  threshold: {args.threshold}")
+  if args.algorithm == "lcs":
+    print(f"  min_density: {args.min_density:.3f}")
+  print(f"  git: {metadata['git_sha']}\n")
+
+  for name in selected:
+    if name not in cases:
+      print(f"  {name}: unknown case, skipping\n")
+      continue
+
+    case = cases[name]
+    source = case["source"]
+    extraction_text = case["extraction_text"]
+    expect_match = case.get("expect_match")
+    n_source_tokens = len(_tokenize_words(source, tokenizer))
+
+    print(f"  {name}: {case['description']}", flush=True)
+    print(f"    source tokens: {n_source_tokens}", flush=True)
+
+    expect_token = case.get("expect_token_interval")
+    expect_char = case.get("expect_char_interval")
+    expect_sub = case.get("expect_substring")
+
+    timings = []
+    last_result = None
+    correctness = "n/a"
+    for i in range(args.runs):
+      print(f"    run {i + 1}/{args.runs}...", end="", flush=True)
+      result = _run_single(
+          aligner,
+          source,
+          extraction_text,
+          tokenizer,
+          args.threshold,
+          args.algorithm,
+          args.min_density,
+      )
+      timings.append(result["elapsed_ms"])
+      last_result = result
+      print(f" {result['elapsed_ms']:.1f}ms", flush=True)
+
+      # Check oracle on every run. All configured expectations are checked.
+      if expect_match is not None:
+        if result["matched"] != expect_match:
+          correctness = "FAIL"
+          print(f"    FAIL: expected matched={expect_match}", flush=True)
+        if expect_token and result["token_interval"]:
+          expected = f"{expect_token[0]}-{expect_token[1]}"
+          if result["token_interval"] != expected:
+            correctness = "FAIL"
+            print(
+                f"    FAIL: token_interval {result['token_interval']}"
+                f" != {expected}",
+                flush=True,
+            )
+        if expect_char and result["char_interval"]:
+          expected = f"{expect_char[0]}-{expect_char[1]}"
+          if result["char_interval"] != expected:
+            correctness = "FAIL"
+            print(
+                f"    FAIL: char_interval {result['char_interval']}"
+                f" != {expected}",
+                flush=True,
+            )
+        if expect_sub and result["matched_substring"] != expect_sub:
+          correctness = "FAIL"
+          print("    FAIL: substring mismatch", flush=True)
+
+    if correctness != "FAIL":
+      correctness = "PASS" if expect_match is not None else "n/a"
+
+    avg_ms = sum(timings) / len(timings)
+    min_ms = min(timings)
+    max_ms = max(timings)
+
+    print(f"    avg: {avg_ms:.1f}ms  min: {min_ms:.1f}ms  max: {max_ms:.1f}ms")
+    print(f"    matched: {last_result['matched']}  correctness: {correctness}")
+    if last_result["matched_substring"]:
+      sub = last_result["matched_substring"]
+      if len(sub) > 80:
+        sub = sub[:80] + "..."
+      print(f"    substring: {sub!r}")
+    print(flush=True)
+
+    results[name] = {
+        "description": case["description"],
+        "source_tokens": n_source_tokens,
+        "runs": args.runs,
+        "avg_ms": round(avg_ms, 2),
+        "min_ms": round(min_ms, 2),
+        "max_ms": round(max_ms, 2),
+        "matched": last_result["matched"],
+        "correctness": correctness,
+        "token_interval": last_result["token_interval"],
+        "char_interval": last_result["char_interval"],
+        "matched_substring": last_result["matched_substring"],
+    }
+
+  if args.json_output:
+    with open(args.json_output, "w") as f:
+      json.dump(results, f, indent=2)
+    print(f"Results written to {args.json_output}")
+
+  return 0
+
+
+if __name__ == "__main__":
+  sys.exit(main())

benchmarks/plotting.py

@@ -0,0 +1,700 @@
+# Copyright 2025 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Visualization generation for benchmark results.
+
+Creates multi-panel plots showing tokenization performance, extraction metrics,
+and cross-language comparisons.
+"""
+
+from datetime import datetime
+import json
+from pathlib import Path
+from typing import Any
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+
+from benchmarks import config
+
+matplotlib.use("Agg")
+plt.style.use(config.DISPLAY.plot_style)
+
+
+def create_diverse_plots(results: dict[str, Any], filepath: Path) -> bool:
+  """Generate comprehensive benchmark visualization.
+
+  Args:
+    results: Benchmark results dictionary with tokenization and extraction data.
+    filepath: Output path for PNG file.
+
+  Returns:
+    True if plot created successfully, False on error.
+  """
+  try:
+    fig = plt.figure(figsize=(15, 10))
+
+    # Create 2x3 grid: tokenization metrics (top), extraction metrics (bottom)
+    gs = fig.add_gridspec(2, 3, hspace=0.25, wspace=0.25)
+
+    ax1 = fig.add_subplot(gs[0, 0])  # Tokenization throughput
+    ax2 = fig.add_subplot(gs[0, 1])  # Token density by language
+    ax3 = fig.add_subplot(gs[0, 2])  # Entity extraction counts
+    ax4 = fig.add_subplot(gs[1, 0])  # Processing speed
+    ax5 = fig.add_subplot(gs[1, 1])  # Summary metrics
+    ax6 = fig.add_subplot(gs[1, 2])  # Unused
+
+    fig.suptitle(
+        f"LangExtract Benchmark - {results['timestamp']}", fontsize=14, y=0.98
+    )
+
+    _plot_tokenization_throughput(ax1, results)
+    _plot_tokenization_rate(ax2, results)
+    _plot_extraction_density(ax3, results)
+    _plot_processing_speed(ax4, results)
+    _plot_summary_table(ax5, results)
+    ax6.axis("off")
+
+    plt.tight_layout(rect=[0, 0.02, 1, 0.96])
+
+    plot_path = filepath.with_suffix(".png")
+    plt.savefig(plot_path, dpi=100, bbox_inches="tight")
+    plt.close()
+
+    print(f"Plot saved to: {plot_path}")
+    return True
+
+  except (IOError, OSError) as e:
+    print(f"Warning: Could not create benchmark plot: {e}")
+    return False
+
+
+def _plot_tokenization_throughput(ax, results):
+  """Plot tokenization throughput (tokens per second) on log scale."""
+  if (
+      config.TOKENIZATION_KEY not in results
+      or not results[config.TOKENIZATION_KEY]
+  ):
+    ax.text(0.5, 0.5, "No tokenization data", ha="center", va="center")
+    ax.set_title("Tokenization Throughput")
+    return
+
+  sizes = [r["words"] for r in results[config.TOKENIZATION_KEY]]
+  speeds = [r["tokens_per_sec"] for r in results[config.TOKENIZATION_KEY]]
+
+  ax.semilogx(sizes, speeds, "b-o", linewidth=2, markersize=8)
+  ax.set_xlabel("Number of Words (log scale)")
+  ax.set_ylabel("Tokens per Second")
+  ax.set_title("Tokenization Throughput")
+  ax.grid(True, alpha=0.3)
+
+  max_speed = max(speeds)
+  ax.set_ylim(0, max_speed * 1.15)
+
+  y_ticks = [0, 100000, 200000, 300000, 400000]
+  ax.set_yticks(y_ticks)
+  ax.set_yticklabels([f"{int(y/1000)}K" if y > 0 else "0" for y in y_ticks])
+
+  for x, y in zip(sizes, speeds):
+    label = f"{y/1000:.0f}K"
+    ax.annotate(
+        label,
+        xy=(x, y),
+        xytext=(0, 5),
+        textcoords="offset points",
+        ha="center",
+        fontsize=9,
+    )
+
+  ax.set_xticks([100, 1000, 10000])
+  ax.set_xticklabels(["10²", "10³", "10⁴"])
+
+
+def _plot_tokenization_rate(ax, results):
+  """Plot tokenization rate by text type."""
+  if config.RESULTS_KEY not in results:
+    ax.text(0.5, 0.5, "No data", ha="center", va="center")
+    ax.set_title("Tokenization Rate")
+    return
+
+  text_types = []
+  tok_per_char = []
+
+  for result in results[config.RESULTS_KEY]:
+    if config.TOKENIZATION_KEY in result and result.get("success", False):
+      text_type = result.get("text_type", "unknown")
+      if text_type not in text_types:
+        text_types.append(text_type)
+        tpc = result[config.TOKENIZATION_KEY]["tokens_per_char"]
+        tok_per_char.append(tpc)
+
+  if not text_types:
+    ax.text(0.5, 0.5, "No tokenization data", ha="center", va="center")
+    ax.set_title("Tokenization Rate")
+    return
+
+  x = np.arange(len(text_types))
+  bars = ax.bar(x, tok_per_char, color="#2196f3", alpha=0.7)
+
+  for bar_rect, val in zip(bars, tok_per_char):
+    ax.text(
+        bar_rect.get_x() + bar_rect.get_width() / 2,
+        val + 0.005,
+        f"{val:.3f}",
+        ha="center",
+        va="bottom",
+        fontsize=9,
+    )
+
+  ax.set_xlabel("Text Type")
+  ax.set_ylabel("Tokens per Character")
+  ax.set_title("Tokenization Rate")
+  ax.set_xticks(x)
+  ax.set_xticklabels([t.capitalize() for t in text_types])
+  ax.grid(True, alpha=0.3, axis="y")
+  ax.set_ylim(0, max(0.30, max(tok_per_char) * 1.2) if tok_per_char else 0.30)
+
+
+def _plot_extraction_density(ax, results):
+  """Plot entity extraction density."""
+  if config.RESULTS_KEY not in results:
+    ax.text(0.5, 0.5, "No data", ha="center", va="center")
+    ax.set_title("Extraction Density")
+    return
+
+  text_types = []
+  densities = []
+
+  for result in results[config.RESULTS_KEY]:
+    if result.get("success", False):
+      text_type = result.get("text_type", "unknown")
+      if text_type not in text_types:
+        text_types.append(text_type)
+
+        char_count = 1000
+        if config.TOKENIZATION_KEY in result:
+          char_count = result[config.TOKENIZATION_KEY].get("num_chars", 1000)
+
+        entity_count = result.get("entity_count", 0)
+        density = (entity_count * 1000) / char_count
+        densities.append(density)
+
+  if not text_types:
+    ax.text(0.5, 0.5, "No successful extractions", ha="center", va="center")
+    ax.set_title("Extraction Density")
+    return
+
+  x = np.arange(len(text_types))
+  bars = ax.bar(x, densities, color="#4caf50", alpha=0.7)
+
+  for bar_rect, val in zip(bars, densities):
+    ax.text(
+        bar_rect.get_x() + bar_rect.get_width() / 2,
+        val,
+        f"{val:.1f}",
+        ha="center",
+        va="bottom",
+        fontsize=9,
+    )
+
+  ax.set_xlabel("Text Type")
+  ax.set_ylabel("Entities per 1K Characters")
+  ax.set_title("Extraction Density")
+  ax.set_xticks(x)
+  ax.set_xticklabels([t.capitalize() for t in text_types])
+  ax.grid(True, alpha=0.3, axis="y")
+
+
+def _plot_processing_speed(ax, results):
+  """Plot processing speed normalized by text size."""
+  if config.RESULTS_KEY not in results:
+    ax.text(0.5, 0.5, "No data", ha="center", va="center")
+    ax.set_title("Processing Speed")
+    return
+
+  text_types = []
+  speeds = []
+
+  for result in results[config.RESULTS_KEY]:
+    if result.get("success", False):
+      text_type = result.get("text_type", "unknown")
+      if text_type not in text_types:
+        text_types.append(text_type)
+
+        char_count = 1000
+        if config.TOKENIZATION_KEY in result:
+          char_count = result[config.TOKENIZATION_KEY].get("num_chars", 1000)
+
+        time_seconds = result.get("time_seconds", 0)
+        speed = (time_seconds * 1000) / char_count
+        speeds.append(speed)
+
+  if not text_types:
+    ax.text(0.5, 0.5, "No timing data", ha="center", va="center")
+    ax.set_title("Processing Speed")
+    return
+
+  x = np.arange(len(text_types))
+  bars = ax.bar(x, speeds, color="#ff9800", alpha=0.7)
+
+  for bar_rect, val in zip(bars, speeds):
+    ax.text(
+        bar_rect.get_x() + bar_rect.get_width() / 2,
+        val,
+        f"{val:.1f}s",
+        ha="center",
+        va="bottom",
+        fontsize=9,
+    )
+
+  ax.set_xlabel("Text Type")
+  ax.set_ylabel("Seconds per 1K Characters")
+  ax.set_title("Processing Speed")
+  ax.set_xticks(x)
+  ax.set_xticklabels([t.capitalize() for t in text_types])
+  ax.grid(True, alpha=0.3, axis="y")
+
+
+def _plot_summary_table(ax, results):
+  """Create a summary of key findings."""
+  ax.axis("off")
+
+  if config.RESULTS_KEY not in results:
+    ax.text(0.5, 0.5, "No data", ha="center", va="center")
+    ax.set_title("Key Metrics")
+    return
+
+  summary_lines = []
+  summary_lines.append("Key Metrics")
+  summary_lines.append("-" * 20)
+  summary_lines.append("")
+
+  success_count = sum(
+      1 for r in results.get(config.RESULTS_KEY, []) if r.get("success")
+  )
+  total_count = len(results.get(config.RESULTS_KEY, []))
+
+  if total_count > 0:
+    summary_lines.append("Tests Run:")
+    summary_lines.append(f"  {success_count} successful")
+    summary_lines.append(f"  {total_count - success_count} failed")
+    summary_lines.append("")
+
+  if success_count > 0:
+    avg_time = (
+        sum(
+            r.get("time_seconds", 0)
+            for r in results.get(config.RESULTS_KEY, [])
+            if r.get("success")
+        )
+        / success_count
+    )
+    summary_lines.append(f"Avg Time: {avg_time:.1f}s")
+
+  summary_text = "\n".join(summary_lines)
+  ax.text(
+      0.5,
+      0.5,
+      summary_text,
+      ha="center",
+      va="center",
+      fontsize=10,
+      family="monospace",
+  )
+
+  ax.set_title("Key Metrics", fontweight="bold", y=0.9)
+
+
+def create_comparison_plots(json_files: list[Path], output_path: Path) -> None:
+  """Create comparison plots from multiple benchmark JSON files.
+
+  Args:
+    json_files: List of paths to benchmark JSON files to compare.
+    output_path: Path where the comparison plot should be saved.
+  """
+  if len(json_files) < 2:
+    print("Need at least 2 JSON files for comparison")
+    return
+
+  all_results = []
+  for json_file in json_files:
+    try:
+      with open(json_file, "r") as f:
+        data = json.load(f)
+        data["filename"] = json_file.stem
+        all_results.append(data)
+    except (IOError, OSError, json.JSONDecodeError) as e:
+      print(f"Error loading {json_file}: {e}")
+      continue
+
+  if len(all_results) < 2:
+    print("Could not load enough valid JSON files for comparison")
+    return
+
+  plt.figure(figsize=(18, 12))
+
+  ax1 = plt.subplot(2, 3, (1, 2))
+  _plot_tokenization_comparison(ax1, all_results)
+
+  ax2 = plt.subplot(2, 3, 3)
+  _plot_entity_comparison(ax2, all_results)
+
+  ax3 = plt.subplot(2, 3, 4)
+  _plot_time_comparison(ax3, all_results)
+
+  ax4 = plt.subplot(2, 3, 5)
+  _plot_success_rate_comparison(ax4, all_results)
+
+  ax5 = plt.subplot(2, 3, 6)
+  _plot_timeline(ax5, all_results)
+
+  timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+  plt.suptitle(
+      f"LangExtract Benchmark Comparison - {timestamp}",
+      fontsize=14,
+      fontweight="bold",
+  )
+  plt.tight_layout(rect=[0, 0.01, 1, 0.95])
+  plt.subplots_adjust(hspace=0.45, wspace=0.35, top=0.93)
+  plt.savefig(output_path, dpi=100, bbox_inches="tight")
+  plt.close()
+  print(f"Comparison plot saved to: {output_path}")
+
+
+def _plot_entity_comparison(ax, all_results):
+  """Plot entity count comparison across runs."""
+  runs = []
+  languages = ["english", "french", "spanish", "japanese"]
+  language_data = []
+
+  for result in all_results:
+    run_name = result["filename"].replace("benchmark_", "")[:10]
+    runs.append(run_name)
+
+    run_counts = {lang: 0 for lang in languages}
+    if config.RESULTS_KEY in result:
+      for res in result[config.RESULTS_KEY]:
+        lang = res.get("text_type", "")
+        if lang in languages and res.get("success"):
+          run_counts[lang] = res.get("entity_count", 0)
+
+    language_data.append(run_counts)
+
+  x = np.arange(len(runs))
+  width = 0.2
+
+  for i, lang in enumerate(languages):
+    counts = [data[lang] for data in language_data]
+    bars = ax.bar(x + i * width, counts, width, label=lang.capitalize())
+
+    for bar_rect, count in zip(bars, counts):
+      if count > 0:
+        ax.text(
+            bar_rect.get_x() + bar_rect.get_width() / 2,
+            bar_rect.get_height() + 0.5,
+            str(count),
+            ha="center",
+            fontsize=7,
+        )
+
+  ax.set_xlabel("Run")
+  ax.set_ylabel("Entity Count")
+  title = "Entities Extracted by Language\n"
+  subtitle = "Number of unique character names found per language"
+  ax.set_title(title, fontweight="bold", fontsize=10)
+  ax.text(
+      0.5,
+      1.01,
+      subtitle,
+      transform=ax.transAxes,
+      ha="center",
+      fontsize=7,
+      style="italic",
+      color="#666666",
+      va="bottom",
+  )
+  ax.set_xticks(x + width * 1.5)
+  ax.set_xticklabels(runs, rotation=45, ha="right")
+  ax.legend(loc="upper left", fontsize=8)
+  ax.grid(True, alpha=0.3)
+  ax.set_ylim(0, ax.get_ylim()[1] * 1.1)
+
+
+def _plot_time_comparison(ax, all_results):
+  """Plot processing time comparison."""
+  runs = []
+  avg_times = []
+
+  for result in all_results:
+    run_name = result["filename"].replace("benchmark_", "")[:10]
+    runs.append(run_name)
+
+    if config.RESULTS_KEY in result:
+      times = [
+          r.get("time_seconds", 0)
+          for r in result[config.RESULTS_KEY]
+          if r.get("success")
+      ]
+      avg_time = sum(times) / len(times) if times else 0
+      avg_times.append(avg_time)
+    else:
+      avg_times.append(0)
+
+  x_pos = np.arange(len(runs))
+  bars = ax.bar(x_pos, avg_times, color="skyblue", edgecolor="navy", alpha=0.7)
+
+  ax.set_xlabel("Run")
+  ax.set_ylabel("Average Time (seconds)")
+  title = "Average Processing Time\n"
+  subtitle = "Mean extraction time across all language tests"
+  ax.set_title(title, fontweight="bold", fontsize=10)
+  ax.text(
+      0.5,
+      1.01,
+      subtitle,
+      transform=ax.transAxes,
+      ha="center",
+      fontsize=7,
+      style="italic",
+      color="#666666",
+      va="bottom",
+  )
+  ax.set_xticks(x_pos)
+  ax.set_xticklabels(runs, rotation=45, ha="right")
+  ax.grid(True, alpha=0.3)
+
+  for bar_rect, time in zip(bars, avg_times):
+    if time > 0:
+      ax.text(
+          bar_rect.get_x() + bar_rect.get_width() / 2,
+          bar_rect.get_height() + 0.1,
+          f"{time:.1f}s",
+          ha="center",
+          fontsize=8,
+      )
+
+  if max(avg_times) > 0:
+    ax.set_ylim(0, max(avg_times) * 1.2)
+
+
+def _plot_tokenization_comparison(ax, all_results):
+  """Plot tokenization throughput comparison as line graphs."""
+
+  for i, result in enumerate(all_results):
+    run_name = result["filename"].replace("benchmark_", "")[:10]
+
+    if config.TOKENIZATION_KEY in result and result[config.TOKENIZATION_KEY]:
+      sizes = [r["words"] for r in result[config.TOKENIZATION_KEY]]
+      speeds = [r["tokens_per_sec"] for r in result[config.TOKENIZATION_KEY]]
+
+      ax.semilogx(
+          sizes,
+          speeds,
+          "o-",
+          linewidth=2,
+          markersize=6,
+          label=run_name,
+          alpha=0.8,
+      )
+
+      for x, y in zip(sizes, speeds):
+        if i == 0:  # Only label first run to avoid overlap
+          label = f"{y/1000:.0f}K"
+          ax.annotate(
+              label,
+              xy=(x, y),
+              xytext=(0, 5),
+              textcoords="offset points",
+              ha="center",
+              fontsize=7,
+          )
+
+  ax.set_xlabel("Number of Words (log scale)")
+  ax.set_ylabel("Tokens per Second")
+  title = "Tokenization Throughput Comparison\n"
+  subtitle = "Speed of text tokenization at different document sizes"
+  ax.set_title(title, fontweight="bold", fontsize=10)
+  ax.text(
+      0.5,
+      1.01,
+      subtitle,
+      transform=ax.transAxes,
+      ha="center",
+      fontsize=7,
+      style="italic",
+      color="#666666",
+      va="bottom",
+  )
+  ax.grid(True, alpha=0.3)
+  ax.legend(loc="best", fontsize=8)
+
+  ax.set_xticks([100, 1000, 10000])
+  ax.set_xticklabels(["10²", "10³", "10⁴"])
+
+  _, ymax = ax.get_ylim()
+  ax.set_ylim(0, ymax * 1.1)
+
+
+def _plot_success_rate_comparison(ax, all_results):
+  """Plot success rate comparison."""
+  runs = []
+  success_rates = []
+
+  for result in all_results:
+    run_name = result["filename"].replace("benchmark_", "")[:10]
+    runs.append(run_name)
+
+    if config.RESULTS_KEY in result:
+      total = len(result[config.RESULTS_KEY])
+      success = sum(1 for r in result[config.RESULTS_KEY] if r.get("success"))
+      rate = (success / total * 100) if total > 0 else 0
+      success_rates.append(rate)
+    else:
+      success_rates.append(0)
+
+  x_pos = np.arange(len(runs))
+  colors = [
+      "green" if rate == 100 else "orange" if rate >= 75 else "red"
+      for rate in success_rates
+  ]
+  bars = ax.bar(x_pos, success_rates, color=colors, alpha=0.7)
+
+  ax.set_xlabel("Run")
+  ax.set_ylabel("Success Rate (%)")
+  title = "Extraction Success Rate\n"
+  subtitle = "Percentage of language tests completed without errors"
+  ax.set_title(title, fontweight="bold", fontsize=10)
+  ax.text(
+      0.5,
+      1.01,
+      subtitle,
+      transform=ax.transAxes,
+      ha="center",
+      fontsize=7,
+      style="italic",
+      color="#666666",
+      va="bottom",
+  )
+  ax.set_ylim(0, 105)
+  ax.set_xticks(x_pos)
+  ax.set_xticklabels(runs, rotation=45, ha="right")
+  ax.axhline(y=100, color="green", linestyle="--", alpha=0.3)
+  ax.grid(True, alpha=0.3)
+
+  for bar_rect, rate in zip(bars, success_rates):
+    ax.text(
+        bar_rect.get_x() + bar_rect.get_width() / 2,
+        bar_rect.get_height() + 1,
+        f"{rate:.0f}%",
+        ha="center",
+        fontsize=8,
+    )
+
+
+def _plot_token_rate_by_language(ax, all_results):
+  """Plot tokenization rates by language."""
+  languages = ["english", "french", "spanish", "japanese"]
+  latest_result = all_results[-1]
+
+  token_rates = []
+  colors = []
+
+  if config.RESULTS_KEY in latest_result:
+    for lang in languages:
+      lang_results = [
+          r
+          for r in latest_result[config.RESULTS_KEY]
+          if r.get("text_type") == lang and r.get("success")
+      ]
+      if lang_results and config.TOKENIZATION_KEY in lang_results[0]:
+        rate = lang_results[0][config.TOKENIZATION_KEY].get(
+            "tokens_per_char", 0
+        )
+        token_rates.append(rate)
+        colors.append(
+            "red" if rate < 0.1 else "orange" if rate < 0.2 else "green"
+        )
+      else:
+        token_rates.append(0)
+        colors.append("gray")
+
+  ax.bar(languages, token_rates, color=colors, alpha=0.7)
+  ax.set_xlabel("Language")
+  ax.set_ylabel("Tokens per Character")
+  ax.set_title("Tokenization Density (Latest Run)")
+  ax.set_xticks(range(len(languages)))
+  ax.set_xticklabels([l.capitalize() for l in languages])
+  ax.grid(True, alpha=0.3)
+
+  for i, (lang, rate) in enumerate(zip(languages, token_rates)):
+    ax.text(i, rate + 0.01, f"{rate:.3f}", ha="center", fontsize=8)
+
+
+def _plot_timeline(ax, all_results):
+  """Plot metrics over time if timestamps available."""
+  timestamps = []
+  entity_totals = []
+
+  for result in all_results:
+    filename = result["filename"]
+    if "timestamp" in result:
+      timestamps.append(result["timestamp"])
+    else:
+      # Try to parse from filename (format: benchmark_YYYYMMDD_HHMMSS)
+      parts = filename.split("_")
+      if len(parts) >= 3:
+        timestamps.append(f"{parts[-2]}_{parts[-1]}")
+      else:
+        timestamps.append(filename[:10])
+
+    if config.RESULTS_KEY in result:
+      total_entities = sum(
+          r.get("entity_count", 0)
+          for r in result[config.RESULTS_KEY]
+          if r.get("success")
+      )
+      entity_totals.append(total_entities)
+    else:
+      entity_totals.append(0)
+
+  x_pos = np.arange(len(timestamps))
+  ax.plot(x_pos, entity_totals, "o-", color="blue", linewidth=2, markersize=8)
+  ax.set_xlabel("Run")
+  ax.set_ylabel("Total Entities")
+  title = "Total Entities Over Time\n"
+  subtitle = "Sum of all entities extracted across all languages"
+  ax.set_title(title, fontweight="bold", fontsize=10)
+  ax.text(
+      0.5,
+      1.01,
+      subtitle,
+      transform=ax.transAxes,
+      ha="center",
+      fontsize=7,
+      style="italic",
+      color="#666666",
+      va="bottom",
+  )
+  ax.set_xticks(x_pos)
+  ax.set_xticklabels([t[-6:] for t in timestamps], rotation=45, ha="right")
+  ax.grid(True, alpha=0.3)
+
+  for i, total in enumerate(entity_totals):
+    ax.text(i, total + 1, str(total), ha="center", fontsize=8)
+
+  if entity_totals:
+    min_val = min(0, min(entity_totals) - 5)
+    max_val = max(entity_totals) + 5
+    ax.set_ylim(min_val, max_val)

benchmarks/utils.py

@@ -0,0 +1,206 @@
+# Copyright 2025 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper functions for benchmark text retrieval and analysis."""
+
+import subprocess
+from typing import Any
+import urllib.error
+import urllib.request
+
+from benchmarks import config
+from langextract.core import tokenizer
+
+
+def download_text(url: str) -> str:
+  """Download text from URL.
+
+  Args:
+    url: URL to download from.
+
+  Returns:
+    Downloaded text content.
+  """
+  try:
+    with urllib.request.urlopen(url) as response:
+      return response.read().decode("utf-8")
+  except (urllib.error.URLError, urllib.error.HTTPError) as e:
+    raise RuntimeError(f"Could not download from {url}: {e}") from e
+
+
+def extract_text_content(full_text: str) -> str:
+  """Extract main content from Gutenberg text.
+
+  Skips headers and footers by taking middle 60% of text.
+
+  Args:
+    full_text: Full text including Gutenberg headers.
+
+  Returns:
+    Extracted main content.
+  """
+  start_marker = "*** START OF"
+  end_marker = "*** END OF"
+
+  start_idx = full_text.upper().find(start_marker)
+  end_idx = full_text.upper().find(end_marker)
+
+  if start_idx != -1 and end_idx != -1:
+    content_start = full_text.find("\n", start_idx) + 1
+
+    # Handle markers with trailing asterisks (e.g., "*** START ... ***").
+    line_end = full_text.find("***", start_idx + 3)
+    if (
+        line_end != -1 and line_end < content_start + 100
+    ):  # Ensure marker is on same line.
+      content_start = full_text.find("\n", line_end) + 1
+
+    return full_text[content_start:end_idx].strip()
+
+  text_length = len(full_text)
+  start = int(text_length * 0.2)
+  end = int(text_length * 0.8)
+  return full_text[start:end].strip()
+
+
+def get_text_from_gutenberg(text_type: config.TextTypes) -> str:
+  """Get text from Project Gutenberg for given language.
+
+  Args:
+    text_type: Type of text (language).
+
+  Returns:
+    Text sample from Gutenberg.
+  """
+  url = config.GUTENBERG_TEXTS[text_type]
+  full_text = download_text(url)
+  content = extract_text_content(full_text)
+
+  mid_point = len(content) // 2
+  start_chunk = max(0, mid_point - 2500)
+  return content[start_chunk : start_chunk + 5000].strip()
+
+
+def get_optimal_text_size(text: str, model_id: str) -> str:
+  """Get optimal text size for model.
+
+  Args:
+    text: Original text.
+    model_id: Model identifier.
+
+  Returns:
+    Text truncated to optimal size.
+  """
+  if (
+      ":" in model_id
+      or "gemma" in model_id.lower()
+      or "llama" in model_id.lower()
+  ):
+    max_chars = 500  # Smaller context for local models.
+  else:
+    max_chars = 5000
+
+  return text[:max_chars]
+
+
+def get_extraction_example(text_type: config.TextTypes) -> dict[str, str]:  # pylint: disable=unused-argument
+  """Get extraction example configuration.
+
+  Args:
+    text_type: Type of text.
+
+  Returns:
+    Dictionary with prompt configuration.
+  """
+  return {
+      "prompt": "Extract all character names from this text",
+  }
+
+
+def get_git_info() -> dict[str, str]:
+  """Get current git branch and commit info.
+
+  Returns:
+    Dictionary with branch and commit info.
+  """
+  try:
+    branch = subprocess.run(
+        ["git", "branch", "--show-current"],
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()
+
+    commit = subprocess.run(
+        ["git", "rev-parse", "--short", "HEAD"],
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()
+
+    status = subprocess.run(
+        ["git", "status", "--porcelain"],
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()
+
+    if status:
+      commit += "-dirty"
+
+    return {"branch": branch, "commit": commit}
+  except subprocess.CalledProcessError:
+    return {"branch": "unknown", "commit": "unknown"}
+
+
+def analyze_tokenization(
+    text: str, tokenizer_inst: tokenizer.Tokenizer | None = None
+) -> dict[str, Any]:
+  """Analyze tokenization of given text.
+
+  Args:
+    text: Text to analyze.
+    tokenizer_inst: Tokenizer instance to use (default: RegexTokenizer).
+
+  Returns:
+    Dictionary with tokenization metrics.
+  """
+  if tokenizer_inst:
+    tokenized = tokenizer_inst.tokenize(text)
+  else:
+    tokenized = tokenizer.tokenize(text)
+  num_tokens = len(tokenized.tokens)
+  num_chars = len(text)
+  tokens_per_char = num_tokens / num_chars if num_chars > 0 else 0
+
+  return {
+      "num_tokens": num_tokens,
+      "num_chars": num_chars,
+      "tokens_per_char": tokens_per_char,
+  }
+
+
+def format_tokenization_summary(analysis: dict[str, Any]) -> str:
+  """Format tokenization analysis as summary string.
+
+  Args:
+    analysis: Tokenization analysis dict.
+
+  Returns:
+    Formatted summary string.
+  """
+  return (
+      f"{analysis['num_tokens']} tokens, "
+      f"{analysis['tokens_per_char']:.3f} tok/char"
+  )