chore: import upstream snapshot with attribution
Auto Update PR / update-prs (push) Has been cancelled
CI / format-check (push) Has been cancelled
CI / test (3.10) (push) Has been cancelled
CI / test (3.11) (push) Has been cancelled
CI / test (3.12) (push) Has been cancelled
CI / live-api-tests (push) Has been cancelled
CI / plugin-integration-test (push) Has been cancelled
CI / ollama-integration-test (push) Has been cancelled
CI / test-fork-pr (push) Has been cancelled
Auto Update PR / update-prs (push) Has been cancelled
CI / format-check (push) Has been cancelled
CI / test (3.10) (push) Has been cancelled
CI / test (3.11) (push) Has been cancelled
CI / test (3.12) (push) Has been cancelled
CI / live-api-tests (push) Has been cancelled
CI / plugin-integration-test (push) Has been cancelled
CI / ollama-integration-test (push) Has been cancelled
CI / test-fork-pr (push) Has been cancelled
This commit is contained in:
Executable
+825
@@ -0,0 +1,825 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2025 Google LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Create a new LangExtract provider plugin with all boilerplate code.
|
||||
|
||||
This script automates steps 1-6 of the provider creation checklist:
|
||||
1. Setup Package Structure
|
||||
2. Configure Entry Point
|
||||
3. Implement Provider
|
||||
4. Add Schema Support (optional)
|
||||
5. Create and run tests
|
||||
6. Generate documentation
|
||||
|
||||
For detailed documentation, see:
|
||||
https://github.com/google/langextract/blob/main/langextract/providers/README.md
|
||||
|
||||
Usage:
|
||||
python create_provider_plugin.py MyProvider
|
||||
python create_provider_plugin.py MyProvider --with-schema
|
||||
python create_provider_plugin.py MyProvider --patterns "^mymodel" "^custom"
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
|
||||
def create_directory_structure(package_name: str, force: bool = False) -> Path:
|
||||
"""Step 1: Setup Package Structure."""
|
||||
print("\n" + "=" * 60)
|
||||
print("STEP 1: Setup Package Structure")
|
||||
print("=" * 60)
|
||||
|
||||
base_dir = Path(f"langextract-{package_name}")
|
||||
package_dir = base_dir / f"langextract_{package_name}"
|
||||
|
||||
if base_dir.exists() and any(base_dir.iterdir()) and not force:
|
||||
print(f"ERROR: {base_dir} already exists and is not empty.")
|
||||
print("Use --force to overwrite or choose a different package name.")
|
||||
sys.exit(1)
|
||||
|
||||
base_dir.mkdir(parents=True, exist_ok=True)
|
||||
package_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"✓ Created directory: {base_dir}/")
|
||||
print(f"✓ Created package: {package_dir}/")
|
||||
print("✅ Step 1 complete: Package structure created")
|
||||
|
||||
return base_dir
|
||||
|
||||
|
||||
def create_pyproject_toml(
|
||||
base_dir: Path, provider_name: str, package_name: str
|
||||
) -> None:
|
||||
"""Step 2: Configure Entry Point."""
|
||||
print("\n" + "=" * 60)
|
||||
print("STEP 2: Configure Entry Point")
|
||||
print("=" * 60)
|
||||
|
||||
content = textwrap.dedent(f"""\
|
||||
[build-system]
|
||||
requires = ["setuptools>=61.0", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "langextract-{package_name}"
|
||||
version = "0.1.0"
|
||||
description = "LangExtract provider plugin for {provider_name}"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
license = {{text = "Apache-2.0"}}
|
||||
dependencies = [
|
||||
"langextract>=1.0.0",
|
||||
# Add your provider's SDK dependencies here
|
||||
]
|
||||
|
||||
[project.entry-points."langextract.providers"]
|
||||
{package_name} = "langextract_{package_name}.provider:{provider_name}LanguageModel"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["langextract_{package_name}*"]
|
||||
""")
|
||||
|
||||
(base_dir / "pyproject.toml").write_text(content, encoding="utf-8")
|
||||
print("✓ Created pyproject.toml with entry point configuration")
|
||||
print("✅ Step 2 complete: Entry point configured")
|
||||
|
||||
|
||||
def create_provider(
|
||||
base_dir: Path,
|
||||
provider_name: str,
|
||||
package_name: str,
|
||||
patterns: list[str],
|
||||
with_schema: bool,
|
||||
) -> None:
|
||||
"""Step 3: Implement Provider."""
|
||||
print("\n" + "=" * 60)
|
||||
print("STEP 3: Implement Provider")
|
||||
print("=" * 60)
|
||||
|
||||
package_dir = base_dir / f"langextract_{package_name}"
|
||||
|
||||
patterns_str = ", ".join(f"r'{p}'" for p in patterns)
|
||||
env_var_safe = re.sub(r"[^A-Z0-9]+", "_", package_name.upper()) + "_API_KEY"
|
||||
|
||||
schema_imports = (
|
||||
f"""
|
||||
from langextract_{package_name}.schema import {provider_name}Schema"""
|
||||
if with_schema
|
||||
else ""
|
||||
)
|
||||
|
||||
schema_init = (
|
||||
"""
|
||||
self.response_schema = kwargs.get('response_schema')
|
||||
self.structured_output = kwargs.get('structured_output', False)"""
|
||||
if with_schema
|
||||
else ""
|
||||
)
|
||||
|
||||
schema_methods = f"""
|
||||
|
||||
@classmethod
|
||||
def get_schema_class(cls):
|
||||
\"\"\"Tell LangExtract about our schema support.\"\"\"
|
||||
from langextract_{package_name}.schema import {provider_name}Schema
|
||||
return {provider_name}Schema
|
||||
|
||||
def apply_schema(self, schema_instance):
|
||||
\"\"\"Apply or clear schema configuration.\"\"\"
|
||||
super().apply_schema(schema_instance)
|
||||
if schema_instance:
|
||||
config = schema_instance.to_provider_config()
|
||||
self.response_schema = config.get('response_schema')
|
||||
self.structured_output = config.get('structured_output', False)
|
||||
else:
|
||||
self.response_schema = None
|
||||
self.structured_output = False""" if with_schema else ""
|
||||
|
||||
schema_infer = (
|
||||
"""
|
||||
api_params = {}
|
||||
if self.response_schema:
|
||||
api_params['response_schema'] = self.response_schema
|
||||
# result = self.client.generate(prompt, **api_params)"""
|
||||
if with_schema
|
||||
else """
|
||||
# result = self.client.generate(prompt, **kwargs)"""
|
||||
)
|
||||
|
||||
provider_content = textwrap.dedent(f'''\
|
||||
"""Provider implementation for {provider_name}."""
|
||||
|
||||
import os
|
||||
import langextract as lx{schema_imports}
|
||||
|
||||
|
||||
@lx.providers.registry.register({patterns_str}, priority=10)
|
||||
class {provider_name}LanguageModel(lx.inference.BaseLanguageModel):
|
||||
"""LangExtract provider for {provider_name}.
|
||||
|
||||
This provider handles model IDs matching: {patterns}
|
||||
"""
|
||||
|
||||
def __init__(self, model_id: str, api_key: str = None, **kwargs):
|
||||
"""Initialize the {provider_name} provider.
|
||||
|
||||
Args:
|
||||
model_id: The model identifier.
|
||||
api_key: API key for authentication.
|
||||
**kwargs: Additional provider-specific parameters.
|
||||
"""
|
||||
super().__init__()
|
||||
self.model_id = model_id
|
||||
self.api_key = api_key or os.environ.get('{env_var_safe}'){schema_init}
|
||||
|
||||
# self.client = YourClient(api_key=self.api_key)
|
||||
self._extra_kwargs = kwargs{schema_methods}
|
||||
|
||||
def infer(self, batch_prompts, **kwargs):
|
||||
"""Run inference on a batch of prompts.
|
||||
|
||||
Args:
|
||||
batch_prompts: List of prompts to process.
|
||||
**kwargs: Additional inference parameters.
|
||||
|
||||
Yields:
|
||||
Lists of ScoredOutput objects, one per prompt.
|
||||
"""
|
||||
for prompt in batch_prompts:{schema_infer}
|
||||
result = f"Mock response for: {{prompt[:50]}}..."
|
||||
yield [lx.inference.ScoredOutput(score=1.0, output=result)]
|
||||
''')
|
||||
|
||||
(package_dir / "provider.py").write_text(provider_content, encoding="utf-8")
|
||||
print("✓ Created provider.py with mock implementation")
|
||||
|
||||
# Create __init__.py
|
||||
init_content = textwrap.dedent(f'''\
|
||||
"""LangExtract provider plugin for {provider_name}."""
|
||||
|
||||
from langextract_{package_name}.provider import {provider_name}LanguageModel
|
||||
|
||||
__all__ = ['{provider_name}LanguageModel']
|
||||
__version__ = "0.1.0"
|
||||
''')
|
||||
|
||||
(package_dir / "__init__.py").write_text(init_content, encoding="utf-8")
|
||||
print("✓ Created __init__.py with exports")
|
||||
print("✅ Step 3 complete: Provider implementation created")
|
||||
|
||||
|
||||
def create_schema(
|
||||
base_dir: Path, provider_name: str, package_name: str
|
||||
) -> None:
|
||||
"""Step 4: Add Schema Support."""
|
||||
print("\n" + "=" * 60)
|
||||
print("STEP 4: Add Schema Support (Optional)")
|
||||
print("=" * 60)
|
||||
|
||||
package_dir = base_dir / f"langextract_{package_name}"
|
||||
|
||||
schema_content = textwrap.dedent(f'''\
|
||||
"""Schema implementation for {provider_name} provider."""
|
||||
|
||||
import langextract as lx
|
||||
from langextract import schema
|
||||
|
||||
|
||||
class {provider_name}Schema(lx.schema.BaseSchema):
|
||||
"""Schema implementation for {provider_name} structured output."""
|
||||
|
||||
def __init__(self, schema_dict: dict):
|
||||
"""Initialize the schema with a dictionary."""
|
||||
self._schema_dict = schema_dict
|
||||
|
||||
@property
|
||||
def schema_dict(self) -> dict:
|
||||
"""Return the schema dictionary."""
|
||||
return self._schema_dict
|
||||
|
||||
@classmethod
|
||||
def from_examples(cls, examples_data, attribute_suffix="_attributes"):
|
||||
"""Build schema from example extractions.
|
||||
|
||||
Args:
|
||||
examples_data: Sequence of ExampleData objects.
|
||||
attribute_suffix: Suffix for attribute fields.
|
||||
|
||||
Returns:
|
||||
A configured {provider_name}Schema instance.
|
||||
"""
|
||||
extraction_types = {{}}
|
||||
for example in examples_data:
|
||||
for extraction in example.extractions:
|
||||
class_name = extraction.extraction_class
|
||||
if class_name not in extraction_types:
|
||||
extraction_types[class_name] = set()
|
||||
if extraction.attributes:
|
||||
extraction_types[class_name].update(extraction.attributes.keys())
|
||||
|
||||
schema_dict = {{
|
||||
"type": "object",
|
||||
"properties": {{
|
||||
"extractions": {{
|
||||
"type": "array",
|
||||
"items": {{"type": "object"}}
|
||||
}}
|
||||
}},
|
||||
"required": ["extractions"]
|
||||
}}
|
||||
|
||||
return cls(schema_dict)
|
||||
|
||||
def to_provider_config(self) -> dict:
|
||||
"""Convert to provider-specific configuration.
|
||||
|
||||
Returns:
|
||||
Dictionary of provider-specific configuration.
|
||||
"""
|
||||
return {{
|
||||
"response_schema": self._schema_dict,
|
||||
"structured_output": True
|
||||
}}
|
||||
|
||||
@property
|
||||
def supports_strict_mode(self) -> bool:
|
||||
"""Whether this schema guarantees valid structured output.
|
||||
|
||||
Returns:
|
||||
True if the provider enforces valid JSON output.
|
||||
"""
|
||||
return False # Set to True only if your provider guarantees valid JSON
|
||||
''')
|
||||
|
||||
(package_dir / "schema.py").write_text(schema_content, encoding="utf-8")
|
||||
print("✓ Created schema.py with BaseSchema implementation")
|
||||
print("✅ Step 4 complete: Schema support added")
|
||||
|
||||
|
||||
def create_test_script(
|
||||
base_dir: Path,
|
||||
provider_name: str,
|
||||
package_name: str,
|
||||
patterns: list[str],
|
||||
with_schema: bool,
|
||||
) -> None:
|
||||
"""Step 5: Create and run tests."""
|
||||
print("\n" + "=" * 60)
|
||||
print("STEP 5: Create Tests")
|
||||
print("=" * 60)
|
||||
|
||||
patterns_literal = "[" + ", ".join(repr(p) for p in patterns) + "]"
|
||||
provider_cls_name = f"{provider_name}LanguageModel"
|
||||
|
||||
test_content = textwrap.dedent(f'''\
|
||||
#!/usr/bin/env python3
|
||||
"""Test script for {provider_name} provider (Step 5 checklist)."""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import langextract as lx
|
||||
from langextract.providers import registry
|
||||
|
||||
try:
|
||||
from langextract_{package_name} import {provider_cls_name}
|
||||
except ImportError:
|
||||
print("ERROR: Plugin not installed. Run: pip install -e .")
|
||||
sys.exit(1)
|
||||
|
||||
lx.providers.load_plugins_once()
|
||||
|
||||
PROVIDER_CLS_NAME = "{provider_cls_name}"
|
||||
PATTERNS = {patterns_literal}
|
||||
|
||||
def _example_id(pattern: str) -> str:
|
||||
\"\"\"Generate test model ID from pattern.\"\"\"
|
||||
base = re.sub(r'^\\^', '', pattern)
|
||||
m = re.match(r"[A-Za-z0-9._-]+", base)
|
||||
base = m.group(0) if m else (base or "model")
|
||||
return f"{{base}}-test"
|
||||
|
||||
sample_ids = [_example_id(p) for p in PATTERNS]
|
||||
sample_ids.append("unknown-model")
|
||||
|
||||
print("Testing {provider_name} Provider - Step 5 Checklist:")
|
||||
print("-" * 50)
|
||||
|
||||
# 1 & 2. Provider registration + pattern matching via resolve()
|
||||
print("1–2. Provider registration & pattern matching")
|
||||
for model_id in sample_ids:
|
||||
try:
|
||||
provider_class = registry.resolve(model_id)
|
||||
ok = provider_class.__name__ == PROVIDER_CLS_NAME
|
||||
status = "✓" if (ok or model_id == "unknown-model") else "✗"
|
||||
note = "expected" if ok else ("expected (no provider)" if model_id == "unknown-model" else "unexpected provider")
|
||||
print(f" {{status}} {{model_id}} -> {{provider_class.__name__ if ok else 'resolved'}} {{note}}")
|
||||
except Exception as e:
|
||||
if model_id == "unknown-model":
|
||||
print(f" ✓ {{model_id}}: No provider found (expected)")
|
||||
else:
|
||||
print(f" ✗ {{model_id}}: resolve() failed: {{e}}")
|
||||
|
||||
# 3. Inference sanity check
|
||||
print("\\n3. Test inference with sample prompts")
|
||||
try:
|
||||
model_id = sample_ids[0] if sample_ids[0] != "unknown-model" else (_example_id(PATTERNS[0]) if PATTERNS else "test-model")
|
||||
provider = {provider_cls_name}(model_id=model_id)
|
||||
prompts = ["Test prompt 1", "Test prompt 2"]
|
||||
results = list(provider.infer(prompts))
|
||||
print(f" ✓ Inference returned {{len(results)}} results")
|
||||
for i, result in enumerate(results):
|
||||
try:
|
||||
out = result[0].output if result and result[0] else None
|
||||
print(f" ✓ Result {{i+1}}: {{(out or '')[:60]}}...")
|
||||
except Exception:
|
||||
print(f" ✗ Result {{i+1}}: Unexpected result shape: {{result}}")
|
||||
except Exception as e:
|
||||
print(f" ✗ ERROR: {{e}}")
|
||||
''')
|
||||
|
||||
if with_schema:
|
||||
test_content += textwrap.dedent(f"""
|
||||
# 4. Test schema creation and application
|
||||
print("\\n4. Test schema creation and application")
|
||||
try:
|
||||
from langextract_{package_name}.schema import {provider_name}Schema
|
||||
from langextract import data
|
||||
|
||||
examples = [
|
||||
data.ExampleData(
|
||||
text="Test text",
|
||||
extractions=[
|
||||
data.Extraction(
|
||||
extraction_class="entity",
|
||||
extraction_text="test",
|
||||
attributes={{"type": "example"}}
|
||||
)
|
||||
]
|
||||
)
|
||||
]
|
||||
|
||||
schema = {provider_name}Schema.from_examples(examples)
|
||||
print(f" ✓ Schema created (keys={{list(schema.schema_dict.keys())}})")
|
||||
|
||||
schema_class = {provider_cls_name}.get_schema_class()
|
||||
print(f" ✓ Provider schema class: {{schema_class.__name__}}")
|
||||
|
||||
provider = {provider_cls_name}(model_id=_example_id(PATTERNS[0]) if PATTERNS else "test-model")
|
||||
provider.apply_schema(schema)
|
||||
print(f" ✓ Schema applied: response_schema={{provider.response_schema is not None}} structured={{getattr(provider, 'structured_output', False)}}")
|
||||
except Exception as e:
|
||||
print(f" ✗ ERROR: {{e}}")
|
||||
""")
|
||||
|
||||
test_content += textwrap.dedent(f"""
|
||||
# 5. Test factory integration
|
||||
print("\\n5. Test factory integration")
|
||||
try:
|
||||
from langextract import factory
|
||||
config = factory.ModelConfig(
|
||||
model_id=_example_id(PATTERNS[0]) if PATTERNS else "test-model",
|
||||
provider="{provider_cls_name}"
|
||||
)
|
||||
model = factory.create_model(config)
|
||||
print(f" ✓ Factory created: {{type(model).__name__}}")
|
||||
except Exception as e:
|
||||
print(f" ✗ ERROR: {{e}}")
|
||||
|
||||
print("\\n" + "-" * 50)
|
||||
print("✅ Testing complete!")
|
||||
""")
|
||||
|
||||
(base_dir / "test_plugin.py").write_text(test_content, encoding="utf-8")
|
||||
print("✓ Created test_plugin.py with comprehensive tests")
|
||||
print("✅ Step 5 complete: Test suite created")
|
||||
|
||||
|
||||
def create_readme(
|
||||
base_dir: Path, provider_name: str, package_name: str, patterns: list[str]
|
||||
) -> None:
|
||||
"""Create README documentation."""
|
||||
print("\n" + "=" * 60)
|
||||
print("STEP 6: Documentation")
|
||||
print("=" * 60)
|
||||
|
||||
def _display(p: str) -> str:
|
||||
"""Strip leading ^ from pattern for display."""
|
||||
return p[1:] if p.startswith("^") else p
|
||||
|
||||
env_var_safe = re.sub(r"[^A-Z0-9]+", "_", package_name.upper()) + "_API_KEY"
|
||||
|
||||
supported = "\n".join(
|
||||
f"- `{_display(p)}*`: Models matching pattern {p}" for p in patterns
|
||||
)
|
||||
|
||||
readme_content = textwrap.dedent(f"""\
|
||||
# LangExtract {provider_name} Provider
|
||||
|
||||
A provider plugin for LangExtract that supports {provider_name} models.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Supported Model IDs
|
||||
|
||||
{supported}
|
||||
|
||||
## Environment Variables
|
||||
|
||||
- `{env_var_safe}`: API key for authentication
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
import langextract as lx
|
||||
|
||||
result = lx.extract(
|
||||
text="Your document here",
|
||||
model_id="{_display(patterns[0]) if patterns else package_name}-model",
|
||||
prompt_description="Extract entities",
|
||||
examples=[...]
|
||||
)
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
1. Install in development mode: `pip install -e .`
|
||||
2. Run tests: `python test_plugin.py`
|
||||
3. Build package: `python -m build`
|
||||
4. Publish to PyPI: `twine upload dist/*`
|
||||
|
||||
## License
|
||||
|
||||
Apache License 2.0
|
||||
""")
|
||||
|
||||
(base_dir / "README.md").write_text(readme_content, encoding="utf-8")
|
||||
print("✓ Created README.md with usage examples")
|
||||
|
||||
|
||||
def create_gitignore(base_dir: Path) -> None:
|
||||
"""Create .gitignore file with Python-specific entries."""
|
||||
gitignore_content = textwrap.dedent("""\
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
.eggs/
|
||||
*.egg
|
||||
|
||||
# Virtual environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
|
||||
# Testing & coverage
|
||||
.pytest_cache/
|
||||
.tox/
|
||||
htmlcov/
|
||||
.coverage
|
||||
.coverage.*
|
||||
|
||||
# Type checking
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
.pytype/
|
||||
|
||||
# IDEs
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# OS-specific
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Temp files
|
||||
*.tmp
|
||||
*.bak
|
||||
*.backup
|
||||
""")
|
||||
|
||||
(base_dir / ".gitignore").write_text(gitignore_content, encoding="utf-8")
|
||||
print("✓ Created .gitignore file with Python-specific entries")
|
||||
|
||||
|
||||
def create_license(base_dir: Path) -> None:
|
||||
"""Create LICENSE file."""
|
||||
license_content = textwrap.dedent("""\
|
||||
# LICENSE
|
||||
|
||||
TODO: Add your license here.
|
||||
|
||||
This is a placeholder license file for your provider plugin.
|
||||
Please replace this with your actual license before distribution.
|
||||
|
||||
Common options include:
|
||||
- Apache License 2.0
|
||||
- MIT License
|
||||
- BSD License
|
||||
- GPL License
|
||||
- Proprietary/Commercial License
|
||||
""")
|
||||
|
||||
(base_dir / "LICENSE").write_text(license_content, encoding="utf-8")
|
||||
print("✓ Created LICENSE file")
|
||||
print("✅ Step 6 complete: Documentation created")
|
||||
|
||||
|
||||
def install_and_test(base_dir: Path) -> bool:
|
||||
"""Install the plugin and run tests."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Installing and testing the plugin...")
|
||||
print("=" * 60)
|
||||
|
||||
os.chdir(base_dir)
|
||||
print("\nInstalling plugin...")
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "-e", "."],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode:
|
||||
print(f"Installation failed: {result.stderr}")
|
||||
return False
|
||||
print("✓ Plugin installed successfully")
|
||||
|
||||
print("\nRunning tests...")
|
||||
result = subprocess.run(
|
||||
[sys.executable, "test_plugin.py"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
print(result.stdout)
|
||||
if result.returncode:
|
||||
print(f"Tests failed: {result.stderr}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
"""Parse command line arguments.
|
||||
|
||||
Returns:
|
||||
Parsed arguments from argparse.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Create a new LangExtract provider plugin",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=textwrap.dedent("""
|
||||
Examples:
|
||||
python create_provider_plugin.py MyProvider
|
||||
python create_provider_plugin.py MyProvider --with-schema
|
||||
python create_provider_plugin.py MyProvider --patterns "^mymodel" "^custom"
|
||||
python create_provider_plugin.py MyProvider --package-name my_custom_name
|
||||
"""),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"provider_name",
|
||||
help="Name of your provider (e.g., MyProvider, CustomLLM)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--patterns",
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="Regex patterns for model IDs (default: based on provider name)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--package-name",
|
||||
default=None,
|
||||
help="Package name (default: lowercase provider name)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--with-schema",
|
||||
action="store_true",
|
||||
help="Include schema support (Step 4)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--no-install", action="store_true", help="Skip installation and testing"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Overwrite existing plugin directory if it exists",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_patterns(patterns: list[str]) -> None:
|
||||
"""Validate regex patterns.
|
||||
|
||||
Args:
|
||||
patterns: List of regex patterns to validate.
|
||||
|
||||
Raises:
|
||||
SystemExit: If any pattern is invalid.
|
||||
"""
|
||||
for p in patterns:
|
||||
try:
|
||||
re.compile(p)
|
||||
except re.error as e:
|
||||
print(f"ERROR: Invalid regex pattern '{p}': {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def print_summary(
|
||||
provider_name: str,
|
||||
package_name: str,
|
||||
patterns: list[str],
|
||||
with_schema: bool,
|
||||
) -> None:
|
||||
"""Print configuration summary.
|
||||
|
||||
Args:
|
||||
provider_name: Name of the provider.
|
||||
package_name: Package name.
|
||||
patterns: List of model ID patterns.
|
||||
with_schema: Whether to include schema support.
|
||||
"""
|
||||
print("\n" + "=" * 60)
|
||||
print("LANGEXTRACT PROVIDER PLUGIN GENERATOR")
|
||||
print("=" * 60)
|
||||
print(f"Provider Name: {provider_name}")
|
||||
print(f"Package Name: langextract-{package_name}")
|
||||
print(f"Model Patterns: {patterns}")
|
||||
print(f"Include Schema: {with_schema}")
|
||||
print("\nFor documentation, see:")
|
||||
print(
|
||||
"https://github.com/google/langextract/blob/main/langextract/providers/README.md"
|
||||
)
|
||||
|
||||
|
||||
def create_plugin(
|
||||
args: argparse.Namespace, package_name: str, patterns: list[str]
|
||||
) -> Path:
|
||||
"""Create the plugin with all necessary files.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments.
|
||||
package_name: Package name.
|
||||
patterns: List of model ID patterns.
|
||||
|
||||
Returns:
|
||||
Path to the created plugin directory.
|
||||
"""
|
||||
base_dir = create_directory_structure(package_name, force=args.force)
|
||||
create_pyproject_toml(base_dir, args.provider_name, package_name)
|
||||
create_provider(
|
||||
base_dir, args.provider_name, package_name, patterns, args.with_schema
|
||||
)
|
||||
|
||||
if args.with_schema:
|
||||
create_schema(base_dir, args.provider_name, package_name)
|
||||
|
||||
create_test_script(
|
||||
base_dir, args.provider_name, package_name, patterns, args.with_schema
|
||||
)
|
||||
create_readme(base_dir, args.provider_name, package_name, patterns)
|
||||
create_gitignore(base_dir)
|
||||
create_license(base_dir)
|
||||
|
||||
return base_dir
|
||||
|
||||
|
||||
def print_completion_summary(with_schema: bool) -> None:
|
||||
"""Print completion summary.
|
||||
|
||||
Args:
|
||||
with_schema: Whether schema support was included.
|
||||
"""
|
||||
print("\n" + "=" * 60)
|
||||
print("SUMMARY: Steps 1-6 Completed")
|
||||
print("=" * 60)
|
||||
print("✅ Package structure created")
|
||||
print("✅ Entry point configured")
|
||||
print("✅ Provider implemented")
|
||||
if with_schema:
|
||||
print("✅ Schema support added")
|
||||
print("✅ Tests created")
|
||||
print("✅ Documentation generated")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the provider plugin generator."""
|
||||
args = parse_arguments()
|
||||
|
||||
package_name = args.package_name or args.provider_name.lower()
|
||||
patterns = args.patterns if args.patterns else [f"^{package_name}"]
|
||||
|
||||
validate_patterns(patterns)
|
||||
print_summary(args.provider_name, package_name, patterns, args.with_schema)
|
||||
|
||||
base_dir = create_plugin(args, package_name, patterns)
|
||||
print_completion_summary(args.with_schema)
|
||||
|
||||
if not args.no_install:
|
||||
success = install_and_test(base_dir)
|
||||
if success:
|
||||
print("\n✅ Plugin created, installed, and tested successfully!")
|
||||
print(f"\nYour plugin is ready at: {base_dir.absolute()}")
|
||||
print("\nNext steps:")
|
||||
print(" 1. Replace mock inference with actual API calls")
|
||||
print(" 2. Update documentation with real examples")
|
||||
print(" 3. Build package: python -m build")
|
||||
print(" 4. Publish to PyPI: twine upload dist/*")
|
||||
else:
|
||||
print(
|
||||
"\n⚠️ Plugin created but tests failed. Please check the"
|
||||
" implementation."
|
||||
)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"\nPlugin created at: {base_dir.absolute()}")
|
||||
print("\nTo install and test:")
|
||||
print(f" cd {base_dir}")
|
||||
print(" pip install -e .")
|
||||
print(" python test_plugin.py")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+209
@@ -0,0 +1,209 @@
|
||||
# Copyright 2025 Google LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""Validation for COMMUNITY_PROVIDERS.md plugin registry table."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import re as regex_module
|
||||
import sys
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
HEADER_ANCHOR = '| Plugin Name | PyPI Package |'
|
||||
END_MARKER = '<!-- ADD NEW PLUGINS ABOVE THIS LINE -->'
|
||||
|
||||
# GitHub username/org and repo patterns
|
||||
GH_NAME = r'[-a-zA-Z0-9]+' # usernames/orgs allow hyphens
|
||||
GH_REPO = r'[-a-zA-Z0-9._]+' # repos allow ., _
|
||||
GH_USER_LINK = rf'\[@{GH_NAME}\]\(https://github\.com/{GH_NAME}\)'
|
||||
GH_MULTI_USER = rf'^{GH_USER_LINK}(,\s*{GH_USER_LINK})*$'
|
||||
|
||||
# Markdown link to a GitHub repo
|
||||
GH_REPO_LINK = rf'^\[[^\]]+\]\(https://github\.com/{GH_NAME}/{GH_REPO}\)$'
|
||||
|
||||
# Issue link must point to LangExtract repository (issues only)
|
||||
LANGEXTRACT_ISSUE_LINK = (
|
||||
r'^\[[^\]]+\]\(https://github\.com/google/langextract/issues/\d+\)$'
|
||||
)
|
||||
|
||||
# PEP 503-ish normalized name (loose): lowercase letters/digits with - _ . separators
|
||||
PYPI_NORMALIZED = r'`[a-z0-9]([\-_.]?[a-z0-9]+)*`'
|
||||
|
||||
MIN_DESC_LEN = 10
|
||||
|
||||
|
||||
def normalize_pypi(name: str) -> str:
|
||||
"""PEP 503 normalization for PyPI package names."""
|
||||
return regex_module.sub(r'[-_.]+', '-', name.strip().lower())
|
||||
|
||||
|
||||
def find_table_bounds(lines: List[str]) -> Tuple[int, int]:
|
||||
start = end = -1
|
||||
for i, line in enumerate(lines):
|
||||
if HEADER_ANCHOR in line:
|
||||
start = i
|
||||
elif start >= 0 and END_MARKER in line:
|
||||
end = i
|
||||
break
|
||||
return start, end
|
||||
|
||||
|
||||
def parse_row(line: str) -> List[str]:
|
||||
# assumes caller trimmed line
|
||||
parts = [c.strip() for c in line.split('|')[1:-1]]
|
||||
return parts
|
||||
|
||||
|
||||
def validate(filepath: Path) -> bool:
|
||||
errors: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
content = filepath.read_text(encoding='utf-8')
|
||||
lines = content.splitlines()
|
||||
|
||||
start, end = find_table_bounds(lines)
|
||||
if start < 0:
|
||||
errors.append('Could not find plugin registry table header.')
|
||||
print_report(errors, warnings)
|
||||
return False
|
||||
if end < 0:
|
||||
errors.append(
|
||||
'Could not find end marker: <!-- ADD NEW PLUGINS ABOVE THIS LINE -->.'
|
||||
)
|
||||
print_report(errors, warnings)
|
||||
return False
|
||||
|
||||
rows: List[Dict] = []
|
||||
seen_names = set()
|
||||
seen_pkgs = set()
|
||||
|
||||
for i in range(start + 2, end):
|
||||
raw = lines[i].strip()
|
||||
if not raw:
|
||||
continue
|
||||
|
||||
if not raw.startswith('|') or not raw.endswith('|'):
|
||||
errors.append(
|
||||
f"Line {i+1}: Not a valid table row (must start and end with '|')."
|
||||
)
|
||||
continue
|
||||
|
||||
cols = parse_row(raw)
|
||||
if len(cols) != 6:
|
||||
errors.append(f'Line {i+1}: Expected 6 columns, found {len(cols)}.')
|
||||
continue
|
||||
|
||||
plugin, pypi, maint, repo, desc, issue_link = cols
|
||||
|
||||
# Basic presence checks
|
||||
if not plugin:
|
||||
errors.append(f'Line {i+1}: Plugin Name is required.')
|
||||
|
||||
if not re.fullmatch(PYPI_NORMALIZED, pypi):
|
||||
errors.append(
|
||||
f'Line {i+1}: PyPI package must be backticked and normalized (e.g.,'
|
||||
' `langextract-provider-foo`).'
|
||||
)
|
||||
elif pypi and not pypi.strip('`').lower().startswith('langextract-'):
|
||||
errors.append(
|
||||
f'Line {i+1}: PyPI package should start with `langextract-` for'
|
||||
' discoverability.'
|
||||
)
|
||||
|
||||
if not re.fullmatch(GH_MULTI_USER, maint):
|
||||
errors.append(
|
||||
f'Line {i+1}: Maintainer must be one or more GitHub handles as links '
|
||||
'(e.g., [@alice](https://github.com/alice) or comma-separated).'
|
||||
)
|
||||
|
||||
if not re.fullmatch(GH_REPO_LINK, repo):
|
||||
errors.append(
|
||||
f'Line {i+1}: GitHub Repo must be a Markdown link to a GitHub'
|
||||
' repository.'
|
||||
)
|
||||
|
||||
if not desc or len(desc) < MIN_DESC_LEN:
|
||||
errors.append(
|
||||
f'Line {i+1}: Description must be at least {MIN_DESC_LEN} characters.'
|
||||
)
|
||||
|
||||
# Issue link is required and must point to LangExtract repo
|
||||
if not issue_link:
|
||||
errors.append(f'Line {i+1}: Issue Link is required.')
|
||||
elif not re.fullmatch(LANGEXTRACT_ISSUE_LINK, issue_link):
|
||||
errors.append(
|
||||
f'Line {i+1}: Issue Link must point to a LangExtract issue (e.g.,'
|
||||
' [#123](https://github.com/google/langextract/issues/123)).'
|
||||
)
|
||||
|
||||
rows.append({
|
||||
'line': i + 1,
|
||||
'plugin': plugin,
|
||||
'pypi': pypi.strip('`').lower() if pypi else '',
|
||||
})
|
||||
|
||||
# Duplicate checks (case-insensitive and PEP 503 normalized)
|
||||
for r in rows:
|
||||
pn_key = r['plugin'].strip().casefold()
|
||||
pk_key = normalize_pypi(r['pypi']) if r['pypi'] else None
|
||||
|
||||
if pn_key in seen_names:
|
||||
errors.append(f"Line {r['line']}: Duplicate Plugin Name '{r['plugin']}'.")
|
||||
seen_names.add(pn_key)
|
||||
|
||||
if pk_key and pk_key in seen_pkgs:
|
||||
errors.append(f"Line {r['line']}: Duplicate PyPI Package '{r['pypi']}'.")
|
||||
if pk_key:
|
||||
seen_pkgs.add(pk_key)
|
||||
|
||||
# Required alphabetical sorting check
|
||||
sorted_by_name = sorted(rows, key=lambda r: r['plugin'].casefold())
|
||||
if [r['plugin'] for r in rows] != [r['plugin'] for r in sorted_by_name]:
|
||||
errors.append('Registry rows must be alphabetically sorted by Plugin Name.')
|
||||
|
||||
# Guardrail: discourage leaving only the example entry
|
||||
if len(rows) == 1 and rows[0]['plugin'].lower().startswith('example'):
|
||||
warnings.append(
|
||||
'The registry currently contains only the example row. Add real'
|
||||
' providers above the marker.'
|
||||
)
|
||||
|
||||
print_report(errors, warnings)
|
||||
return not errors
|
||||
|
||||
|
||||
def print_report(errors: List[str], warnings: List[str]) -> None:
|
||||
if errors:
|
||||
print('❌ Validation failed:')
|
||||
for e in errors:
|
||||
print(f' • {e}')
|
||||
if warnings:
|
||||
print('⚠️ Warnings:')
|
||||
for w in warnings:
|
||||
print(f' • {w}')
|
||||
if not errors and not warnings:
|
||||
print('✅ Table format validation passed!')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
path = Path('COMMUNITY_PROVIDERS.md')
|
||||
if len(sys.argv) > 1:
|
||||
path = Path(sys.argv[1])
|
||||
if not path.exists():
|
||||
print(f'❌ Error: File not found: {path}')
|
||||
sys.exit(1)
|
||||
ok = validate(path)
|
||||
sys.exit(0 if ok else 1)
|
||||
Reference in New Issue
Block a user