chore: import upstream snapshot with attribution
Dashboard / frontend (push) Failing after 0s
Dashboard / api (push) Failing after 0s
Lint PowerShell / powershell-lint (ubuntu-latest) (push) Failing after 1s
Python Lint / Lint Python with Ruff (push) Failing after 1s
ShellCheck / Lint shell scripts (push) Failing after 1s
Matrix Smoke / linux-smoke (push) Failing after 1s
Matrix Smoke / distro: cachyos (push) Failing after 15s
Matrix Smoke / distro: linux-mint-21.3 (push) Failing after 15s
Matrix Smoke / distro: debian-12 (push) Failing after 5m21s
Matrix Smoke / distro: fedora-41 (push) Failing after 4m56s
Matrix Smoke / distro: ubuntu-24.04 (push) Failing after 2m13s
Matrix Smoke / distro: rocky-9 (push) Failing after 10m39s
Matrix Smoke / distro: manjaro (push) Failing after 12m11s
Matrix Smoke / distro: opensuse-tw (push) Failing after 11m53s
Matrix Smoke / distro: archlinux (push) Failing after 20m3s
Matrix Smoke / distro: ubuntu-22.04 (push) Failing after 13m49s
Validate .env Schema / tier-1-env-validation (push) Successful in 52s
Validate .env Schema / tier-2-env-validation (push) Successful in 44s
Validate .env Schema / tier-3-env-validation (push) Successful in 52s
Validate .env Schema / tier-4-env-validation (push) Successful in 51s
Validate Extensions Catalog / Check catalog is up-to-date (push) Failing after 9m47s
Secret Scan / Scan for secrets (push) Failing after 21m4s
Validate Docker Compose / Validate Docker Compose files (push) Has been cancelled
Python Type Check / Type check with mypy (push) Has been cancelled
Validate .env Schema / tier-0-env-validation (push) Has been cancelled
Test Linux / integration-smoke (push) Has been cancelled
Lint PowerShell / powershell-lint (windows-latest) (push) Has been cancelled
Matrix Smoke / macos-smoke (push) Has been cancelled
Dashboard / frontend (push) Failing after 0s
Dashboard / api (push) Failing after 0s
Lint PowerShell / powershell-lint (ubuntu-latest) (push) Failing after 1s
Python Lint / Lint Python with Ruff (push) Failing after 1s
ShellCheck / Lint shell scripts (push) Failing after 1s
Matrix Smoke / linux-smoke (push) Failing after 1s
Matrix Smoke / distro: cachyos (push) Failing after 15s
Matrix Smoke / distro: linux-mint-21.3 (push) Failing after 15s
Matrix Smoke / distro: debian-12 (push) Failing after 5m21s
Matrix Smoke / distro: fedora-41 (push) Failing after 4m56s
Matrix Smoke / distro: ubuntu-24.04 (push) Failing after 2m13s
Matrix Smoke / distro: rocky-9 (push) Failing after 10m39s
Matrix Smoke / distro: manjaro (push) Failing after 12m11s
Matrix Smoke / distro: opensuse-tw (push) Failing after 11m53s
Matrix Smoke / distro: archlinux (push) Failing after 20m3s
Matrix Smoke / distro: ubuntu-22.04 (push) Failing after 13m49s
Validate .env Schema / tier-1-env-validation (push) Successful in 52s
Validate .env Schema / tier-2-env-validation (push) Successful in 44s
Validate .env Schema / tier-3-env-validation (push) Successful in 52s
Validate .env Schema / tier-4-env-validation (push) Successful in 51s
Validate Extensions Catalog / Check catalog is up-to-date (push) Failing after 9m47s
Secret Scan / Scan for secrets (push) Failing after 21m4s
Validate Docker Compose / Validate Docker Compose files (push) Has been cancelled
Python Type Check / Type check with mypy (push) Has been cancelled
Validate .env Schema / tier-0-env-validation (push) Has been cancelled
Test Linux / integration-smoke (push) Has been cancelled
Lint PowerShell / powershell-lint (windows-latest) (push) Has been cancelled
Matrix Smoke / macos-smoke (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Shared helper for Anthropic API authentication.
|
||||
|
||||
Provides a unified `create_message()` function using the Anthropic Python SDK.
|
||||
Used by scanner scripts: generate-type-hints.py, generate-docstrings.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContentBlock:
|
||||
type: str
|
||||
text: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MessageResponse:
|
||||
"""Minimal response object matching anthropic.Message interface used by scanner scripts."""
|
||||
|
||||
content: list[ContentBlock] = field(default_factory=list)
|
||||
usage: Usage = field(default_factory=lambda: Usage(0, 0))
|
||||
|
||||
|
||||
def create_message(
|
||||
*,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
messages: list[dict[str, Any]],
|
||||
thinking: dict[str, Any] | None = None,
|
||||
) -> MessageResponse:
|
||||
"""Create a message using the Anthropic API."""
|
||||
import anthropic
|
||||
|
||||
api_key = os.getenv("ANTHROPIC_API_KEY")
|
||||
if not api_key:
|
||||
print(
|
||||
"::error::No API credentials. Set ANTHROPIC_API_KEY",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
client = anthropic.Anthropic(api_key=api_key)
|
||||
|
||||
kwargs: dict[str, Any] = dict(model=model, max_tokens=max_tokens, messages=messages)
|
||||
if thinking:
|
||||
kwargs["thinking"] = thinking
|
||||
kwargs["temperature"] = 1
|
||||
else:
|
||||
kwargs["temperature"] = temperature
|
||||
|
||||
response = client.messages.create(**kwargs)
|
||||
|
||||
content_blocks = [
|
||||
ContentBlock(type="text", text=block.text)
|
||||
for block in response.content
|
||||
if block.type == "text"
|
||||
]
|
||||
|
||||
return MessageResponse(
|
||||
content=content_blocks,
|
||||
usage=Usage(
|
||||
input_tokens=response.usage.input_tokens,
|
||||
output_tokens=response.usage.output_tokens,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Apply Docstrings from suggestions JSON to source files.
|
||||
|
||||
Reads /tmp/documentation-suggestions.json (generated by generate-docstrings.py),
|
||||
inserts Google-style docstrings into source files using AST-based function lookup.
|
||||
|
||||
Safety: validates each file with py_compile after modification; reverts on failure.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
import py_compile
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROTECTED_PATTERNS = [
|
||||
".github/workflows/",
|
||||
".env",
|
||||
"ods/installers/",
|
||||
"ods/ods-cli",
|
||||
"ods/config/",
|
||||
]
|
||||
|
||||
|
||||
def is_protected(file_path: str) -> bool:
|
||||
"""Check if a file path matches any protected patterns."""
|
||||
for pattern in PROTECTED_PATTERNS:
|
||||
if file_path.startswith(pattern) or f"/{pattern}" in file_path:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def find_function_info(source: str, function_name: str) -> dict | None:
|
||||
"""Use AST to find function line and check if it already has a docstring."""
|
||||
try:
|
||||
tree = ast.parse(source)
|
||||
except SyntaxError:
|
||||
return None
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
if node.name == function_name:
|
||||
has_docstring = ast.get_docstring(node) is not None
|
||||
return {
|
||||
"lineno": node.lineno,
|
||||
"has_docstring": has_docstring,
|
||||
"body_start": node.body[0].lineno if node.body else node.lineno + 1,
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def find_def_end_line(lines: list[str], start_idx: int) -> int:
|
||||
"""Find the 0-based index of the last line of a def statement."""
|
||||
depth = 0
|
||||
for i in range(start_idx, len(lines)):
|
||||
line = lines[i]
|
||||
depth += line.count("(") - line.count(")")
|
||||
if depth <= 0 and ":" in line:
|
||||
stripped = line.rstrip()
|
||||
if stripped.endswith(":"):
|
||||
return i
|
||||
colon_pos = stripped.rfind(":")
|
||||
if colon_pos >= 0:
|
||||
return i
|
||||
return start_idx
|
||||
|
||||
|
||||
def get_body_indent(lines: list[str], def_end_idx: int) -> str:
|
||||
"""Determine the indentation level of the function body."""
|
||||
for i in range(def_end_idx + 1, min(def_end_idx + 5, len(lines))):
|
||||
line = lines[i]
|
||||
stripped = line.strip()
|
||||
if stripped and not stripped.startswith("#"):
|
||||
return line[: len(line) - len(line.lstrip())]
|
||||
|
||||
def_line = lines[def_end_idx] if def_end_idx < len(lines) else ""
|
||||
def_indent = def_line[: len(def_line) - len(def_line.lstrip())]
|
||||
return def_indent + " "
|
||||
|
||||
|
||||
def format_docstring(docstring: str, indent: str) -> list[str]:
|
||||
"""Format a docstring with proper indentation as lines to insert."""
|
||||
doc_lines = docstring.split("\n")
|
||||
|
||||
if len(doc_lines) == 1:
|
||||
return [f'{indent}"""{doc_lines[0]}"""\n']
|
||||
|
||||
result = [f'{indent}"""{doc_lines[0]}\n']
|
||||
for line in doc_lines[1:]:
|
||||
if line.strip():
|
||||
result.append(f"{indent}{line}\n")
|
||||
else:
|
||||
result.append("\n")
|
||||
result.append(f'{indent}"""\n')
|
||||
return result
|
||||
|
||||
|
||||
def apply_docstrings(suggestions_path: str) -> dict:
|
||||
"""Apply generated docstring suggestions to Python source files."""
|
||||
with open(suggestions_path, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
functions = data.get("functions_documented", [])
|
||||
if not functions:
|
||||
print("No docstring suggestions to apply.")
|
||||
return {
|
||||
"files_modified": 0,
|
||||
"docstrings_inserted": 0,
|
||||
"files_reverted": 0,
|
||||
"skipped_existing": 0,
|
||||
}
|
||||
|
||||
by_file: dict[str, list] = {}
|
||||
for func in functions:
|
||||
file_path = func.get("file", "")
|
||||
if not file_path:
|
||||
continue
|
||||
if is_protected(file_path):
|
||||
print(f" Skipping protected file: {file_path}")
|
||||
continue
|
||||
if not Path(file_path).exists():
|
||||
print(f" Skipping missing file: {file_path}")
|
||||
continue
|
||||
by_file.setdefault(file_path, []).append(func)
|
||||
|
||||
files_modified = 0
|
||||
docstrings_inserted = 0
|
||||
files_reverted = 0
|
||||
skipped_existing = 0
|
||||
|
||||
for file_path, file_funcs in by_file.items():
|
||||
print(f"\nProcessing {file_path} ({len(file_funcs)} functions)...")
|
||||
|
||||
original_content = Path(file_path).read_text()
|
||||
source = original_content
|
||||
lines = source.splitlines(keepends=True)
|
||||
|
||||
located = []
|
||||
for func in file_funcs:
|
||||
info = find_function_info(source, func["function"])
|
||||
if info is None:
|
||||
print(
|
||||
f" Could not find function '{func['function']}' in AST, skipping"
|
||||
)
|
||||
continue
|
||||
if info["has_docstring"]:
|
||||
print(
|
||||
f" Function '{func['function']}' already has docstring, skipping"
|
||||
)
|
||||
skipped_existing += 1
|
||||
continue
|
||||
located.append((info["lineno"], func))
|
||||
|
||||
located.sort(key=lambda x: x[0], reverse=True)
|
||||
|
||||
inserted_in_file = 0
|
||||
for actual_line, func in located:
|
||||
docstring = func.get("docstring", "").strip()
|
||||
if not docstring:
|
||||
continue
|
||||
|
||||
start_idx = actual_line - 1
|
||||
if start_idx >= len(lines) or start_idx < 0:
|
||||
continue
|
||||
|
||||
def_end_idx = find_def_end_line(lines, start_idx)
|
||||
body_indent = get_body_indent(lines, def_end_idx)
|
||||
doc_lines = format_docstring(docstring, body_indent)
|
||||
|
||||
insert_point = def_end_idx + 1
|
||||
lines[insert_point:insert_point] = doc_lines
|
||||
inserted_in_file += 1
|
||||
print(
|
||||
f" Inserted docstring: {func['function']} (after line {def_end_idx + 1})"
|
||||
)
|
||||
|
||||
if inserted_in_file == 0:
|
||||
continue
|
||||
|
||||
new_source = "".join(lines)
|
||||
Path(file_path).write_text(new_source)
|
||||
|
||||
try:
|
||||
py_compile.compile(file_path, doraise=True)
|
||||
files_modified += 1
|
||||
docstrings_inserted += inserted_in_file
|
||||
print(f" Validated: {file_path} ({inserted_in_file} docstrings)")
|
||||
except py_compile.PyCompileError as e:
|
||||
print(f" REVERT: {file_path} failed compilation: {e}")
|
||||
Path(file_path).write_text(original_content)
|
||||
files_reverted += 1
|
||||
|
||||
return {
|
||||
"files_modified": files_modified,
|
||||
"docstrings_inserted": docstrings_inserted,
|
||||
"files_reverted": files_reverted,
|
||||
"skipped_existing": skipped_existing,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Load and apply generated docstrings."""
|
||||
suggestions_path = (
|
||||
sys.argv[1] if len(sys.argv) > 1 else "/tmp/documentation-suggestions.json"
|
||||
)
|
||||
|
||||
if not Path(suggestions_path).exists():
|
||||
print(f"Suggestions file not found: {suggestions_path}")
|
||||
print("No docstrings to apply.")
|
||||
return
|
||||
|
||||
print(f"Loading suggestions from {suggestions_path}...")
|
||||
result = apply_docstrings(suggestions_path)
|
||||
|
||||
print("\n## Documentation Application Summary\n")
|
||||
print(f"- **Files modified**: {result['files_modified']}")
|
||||
print(f"- **Docstrings inserted**: {result['docstrings_inserted']}")
|
||||
print(f"- **Skipped** (already had docstring): {result['skipped_existing']}")
|
||||
print(f"- **Files reverted** (compilation failed): {result['files_reverted']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,245 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Apply Type Hints from suggestions JSON to source files.
|
||||
|
||||
Reads /tmp/type-hints-suggestions.json (generated by generate-type-hints.py),
|
||||
applies typed signatures to source files using AST-based function lookup.
|
||||
|
||||
Safety: validates each file with py_compile after modification; reverts on failure.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
import py_compile
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROTECTED_PATTERNS = [
|
||||
".github/workflows/",
|
||||
".env",
|
||||
"ods/installers/",
|
||||
"ods/ods-cli",
|
||||
"ods/config/",
|
||||
]
|
||||
|
||||
|
||||
def is_protected(file_path: str) -> bool:
|
||||
"""Check if a file path matches any protected patterns."""
|
||||
for pattern in PROTECTED_PATTERNS:
|
||||
if file_path.startswith(pattern) or f"/{pattern}" in file_path:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def find_function_line(source: str, function_name: str) -> int | None:
|
||||
"""Use AST to find the actual line number of a function by name."""
|
||||
try:
|
||||
tree = ast.parse(source)
|
||||
except SyntaxError:
|
||||
return None
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
if node.name == function_name:
|
||||
return node.lineno
|
||||
return None
|
||||
|
||||
|
||||
def find_def_end_line(lines: list[str], start_idx: int) -> int:
|
||||
"""Find the line index where a def statement ends (the line with the colon)."""
|
||||
depth = 0
|
||||
for i in range(start_idx, len(lines)):
|
||||
line = lines[i]
|
||||
depth += line.count("(") - line.count(")")
|
||||
if depth <= 0 and ":" in line:
|
||||
stripped = line.rstrip()
|
||||
if stripped.endswith(":"):
|
||||
return i
|
||||
colon_pos = stripped.rfind(":")
|
||||
if colon_pos >= 0:
|
||||
return i
|
||||
return start_idx
|
||||
|
||||
|
||||
def get_existing_imports(source: str) -> set[str]:
|
||||
"""Extract all existing import statements from source."""
|
||||
imports = set()
|
||||
for line in source.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("import ") or stripped.startswith("from "):
|
||||
imports.add(stripped)
|
||||
return imports
|
||||
|
||||
|
||||
def find_last_import_line(lines: list[str]) -> int:
|
||||
"""Find the 0-based index of the last import line in the file."""
|
||||
last_import = -1
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("import ") or stripped.startswith("from "):
|
||||
last_import = i
|
||||
return last_import
|
||||
|
||||
|
||||
def normalize_import(imp: str) -> list[str]:
|
||||
"""Normalize an import statement."""
|
||||
return [imp.strip()]
|
||||
|
||||
|
||||
def apply_type_hints(suggestions_path: str) -> dict:
|
||||
"""Apply generated type hint suggestions to Python function signatures."""
|
||||
with open(suggestions_path, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
functions = data.get("functions_annotated", [])
|
||||
if not functions:
|
||||
print("No type hint suggestions to apply.")
|
||||
return {"files_modified": 0, "functions_applied": 0, "files_reverted": 0}
|
||||
|
||||
by_file: dict[str, list] = {}
|
||||
for func in functions:
|
||||
file_path = func.get("file", "")
|
||||
if not file_path:
|
||||
continue
|
||||
if is_protected(file_path):
|
||||
print(f" Skipping protected file: {file_path}")
|
||||
continue
|
||||
if not Path(file_path).exists():
|
||||
print(f" Skipping missing file: {file_path}")
|
||||
continue
|
||||
by_file.setdefault(file_path, []).append(func)
|
||||
|
||||
files_modified = 0
|
||||
functions_applied = 0
|
||||
files_reverted = 0
|
||||
|
||||
for file_path, file_funcs in by_file.items():
|
||||
print(f"\nProcessing {file_path} ({len(file_funcs)} functions)...")
|
||||
|
||||
original_content = Path(file_path).read_text()
|
||||
source = original_content
|
||||
lines = source.splitlines(keepends=True)
|
||||
|
||||
located = []
|
||||
for func in file_funcs:
|
||||
actual_line = find_function_line(source, func["function"])
|
||||
if actual_line is None:
|
||||
print(
|
||||
f" Could not find function '{func['function']}' in AST, skipping"
|
||||
)
|
||||
continue
|
||||
located.append((actual_line, func))
|
||||
|
||||
located.sort(key=lambda x: x[0], reverse=True)
|
||||
|
||||
applied_in_file = 0
|
||||
for actual_line, func in located:
|
||||
typed_sig = func.get("typed_signature", "").strip()
|
||||
if not typed_sig:
|
||||
continue
|
||||
|
||||
start_idx = actual_line - 1
|
||||
if start_idx >= len(lines) or start_idx < 0:
|
||||
continue
|
||||
|
||||
end_idx = find_def_end_line(lines, start_idx)
|
||||
|
||||
current_line = lines[start_idx]
|
||||
indent = current_line[: len(current_line) - len(current_line.lstrip())]
|
||||
|
||||
current_stripped = current_line.lstrip()
|
||||
is_async = current_stripped.startswith("async ")
|
||||
|
||||
typed_stripped = typed_sig.lstrip()
|
||||
if is_async and not typed_stripped.startswith("async "):
|
||||
typed_sig = "async " + typed_sig
|
||||
elif not is_async and typed_stripped.startswith("async "):
|
||||
typed_sig = typed_sig.replace("async ", "", 1)
|
||||
|
||||
if not typed_sig.rstrip().endswith(":"):
|
||||
typed_sig = typed_sig.rstrip() + ":"
|
||||
|
||||
new_line = indent + typed_sig.strip() + "\n"
|
||||
|
||||
lines[start_idx : end_idx + 1] = [new_line]
|
||||
applied_in_file += 1
|
||||
print(f" Applied: {func['function']} (line {actual_line})")
|
||||
|
||||
if applied_in_file == 0:
|
||||
continue
|
||||
|
||||
new_source = "".join(lines)
|
||||
existing_imports = get_existing_imports(new_source)
|
||||
|
||||
imports_to_add = []
|
||||
for func in file_funcs:
|
||||
for imp in func.get("imports_needed", []):
|
||||
for normalized in normalize_import(imp):
|
||||
if normalized not in existing_imports:
|
||||
imports_to_add.append(normalized)
|
||||
existing_imports.add(normalized)
|
||||
|
||||
if imports_to_add:
|
||||
lines = new_source.splitlines(keepends=True)
|
||||
insert_idx = find_last_import_line(lines)
|
||||
if insert_idx >= 0:
|
||||
insert_point = insert_idx + 1
|
||||
else:
|
||||
insert_point = 0
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
if (
|
||||
stripped.startswith("#")
|
||||
or stripped.startswith('"""')
|
||||
or stripped.startswith("'''")
|
||||
or not stripped
|
||||
):
|
||||
insert_point = i + 1
|
||||
else:
|
||||
break
|
||||
|
||||
import_lines = [imp + "\n" for imp in imports_to_add]
|
||||
lines[insert_point:insert_point] = import_lines
|
||||
new_source = "".join(lines)
|
||||
|
||||
Path(file_path).write_text(new_source)
|
||||
|
||||
try:
|
||||
py_compile.compile(file_path, doraise=True)
|
||||
files_modified += 1
|
||||
functions_applied += applied_in_file
|
||||
print(f" Validated: {file_path} ({applied_in_file} functions)")
|
||||
except py_compile.PyCompileError as e:
|
||||
print(f" REVERT: {file_path} failed compilation: {e}")
|
||||
Path(file_path).write_text(original_content)
|
||||
files_reverted += 1
|
||||
|
||||
return {
|
||||
"files_modified": files_modified,
|
||||
"functions_applied": functions_applied,
|
||||
"files_reverted": files_reverted,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Load and apply generated type hints."""
|
||||
suggestions_path = (
|
||||
sys.argv[1] if len(sys.argv) > 1 else "/tmp/type-hints-suggestions.json"
|
||||
)
|
||||
|
||||
if not Path(suggestions_path).exists():
|
||||
print(f"Suggestions file not found: {suggestions_path}")
|
||||
print("No type hints to apply.")
|
||||
return
|
||||
|
||||
print(f"Loading suggestions from {suggestions_path}...")
|
||||
result = apply_type_hints(suggestions_path)
|
||||
|
||||
print("\n## Type Hints Application Summary\n")
|
||||
print(f"- **Files modified**: {result['files_modified']}")
|
||||
print(f"- **Functions updated**: {result['functions_applied']}")
|
||||
print(f"- **Files reverted** (compilation failed): {result['files_reverted']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Docstring Generator using Claude Haiku 4.5
|
||||
|
||||
Generates Google-style docstrings for functions missing documentation.
|
||||
Processes in batches of 8 functions per API call for cost efficiency.
|
||||
|
||||
Cost: $20-35 per run (depending on function count and complexity)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from anthropic_helper import create_message
|
||||
|
||||
|
||||
def load_missing_docs(docs_path: Path) -> list[dict[str, Any]]:
|
||||
"""Load functions without docstrings from JSON."""
|
||||
try:
|
||||
with open(docs_path, "r") as f:
|
||||
docs = json.load(f)
|
||||
return docs
|
||||
except FileNotFoundError:
|
||||
print(f"::error::Docs file not found: {docs_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except json.JSONDecodeError:
|
||||
print(f"::error::Invalid JSON in docs file: {docs_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def read_function_code(file_path: str, function_name: str, line: int) -> str:
|
||||
"""Read the function code from file."""
|
||||
try:
|
||||
with open(file_path, "r") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
start_line = line - 1
|
||||
code_lines = []
|
||||
indent_level = None
|
||||
|
||||
for i in range(start_line, len(lines)):
|
||||
line_text = lines[i]
|
||||
|
||||
if indent_level is None:
|
||||
indent_level = len(line_text) - len(line_text.lstrip())
|
||||
|
||||
current_indent = len(line_text) - len(line_text.lstrip())
|
||||
if i > start_line and current_indent <= indent_level and line_text.strip():
|
||||
if line_text.strip().startswith(("def ", "class ", "@")):
|
||||
break
|
||||
|
||||
code_lines.append(line_text)
|
||||
|
||||
if len(code_lines) >= 50:
|
||||
break
|
||||
|
||||
return "".join(code_lines)
|
||||
except Exception as e:
|
||||
return f"# Could not read function code: {e}"
|
||||
|
||||
|
||||
def build_docstring_prompt(functions: list[dict[str, Any]]) -> str:
|
||||
"""Build the prompt for generating docstrings."""
|
||||
functions_formatted = []
|
||||
|
||||
for i, func in enumerate(functions, 1):
|
||||
code = read_function_code(func["file"], func["function"], func["line"])
|
||||
functions_formatted.append(f"""
|
||||
### Function {i}: `{func["function"]}` in `{func["file"]}`
|
||||
|
||||
```python
|
||||
{code}
|
||||
```
|
||||
""")
|
||||
|
||||
functions_text = "\n".join(functions_formatted)
|
||||
|
||||
return f"""You are a Python documentation expert. Generate clear, helpful Google-style docstrings for functions missing documentation.
|
||||
|
||||
## Functions to Document ({len(functions)} total)
|
||||
|
||||
{functions_text}
|
||||
|
||||
## Your Task
|
||||
|
||||
For each function, provide a complete Google-style docstring including:
|
||||
1. **Summary line** - One sentence describing what the function does
|
||||
2. **Args section** - Document each parameter with type and description
|
||||
3. **Returns section** - Document return value with type and description
|
||||
4. **Raises section** - Document exceptions raised (if applicable)
|
||||
5. **Examples section** (optional) - Usage examples for complex functions
|
||||
|
||||
## Guidelines
|
||||
|
||||
- Summary line: Start with imperative verb (e.g., "Calculate", "Return", "Process")
|
||||
- Be concise but informative
|
||||
- Don't repeat the function name in the summary
|
||||
- Use present tense for descriptions
|
||||
- Include type information in Args/Returns even if type hints exist
|
||||
- Only include Raises section if function actually raises exceptions
|
||||
- Only include Example section for non-trivial functions
|
||||
|
||||
## Output Format
|
||||
|
||||
Respond with JSON:
|
||||
|
||||
```json
|
||||
{{
|
||||
"functions_documented": [
|
||||
{{
|
||||
"file": "app/example.py",
|
||||
"function": "process_data",
|
||||
"line": 42,
|
||||
"docstring": "Process data items and return results.\\n\\nArgs:\\n data: List of items to process.\\n options: Optional configuration dict.\\n\\nReturns:\\n Processed results as dict."
|
||||
}}
|
||||
],
|
||||
"summary": {{
|
||||
"total_documented": 5,
|
||||
"functions_with_examples": 2,
|
||||
"functions_with_raises": 3
|
||||
}}
|
||||
}}
|
||||
```
|
||||
|
||||
Begin your analysis."""
|
||||
|
||||
|
||||
def generate_docstrings_batch(
|
||||
functions: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
"""Generate docstrings for a batch of functions using Claude Haiku 4.5."""
|
||||
prompt = build_docstring_prompt(functions)
|
||||
|
||||
try:
|
||||
response = create_message(
|
||||
model="claude-haiku-4-5-20251001",
|
||||
max_tokens=4096,
|
||||
temperature=0,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
)
|
||||
|
||||
response_text = ""
|
||||
for block in response.content:
|
||||
if block.type == "text":
|
||||
response_text += block.text
|
||||
|
||||
json_start = response_text.find("```json")
|
||||
if json_start != -1:
|
||||
json_start = response_text.find("\n", json_start) + 1
|
||||
json_end = response_text.find("```", json_start)
|
||||
response_text = response_text[json_start:json_end].strip()
|
||||
else:
|
||||
json_start = response_text.find("{")
|
||||
json_end = response_text.rfind("}") + 1
|
||||
if json_start != -1 and json_end > json_start:
|
||||
response_text = response_text[json_start:json_end]
|
||||
|
||||
result = json.loads(response_text)
|
||||
|
||||
result["_metadata"] = {
|
||||
"model": "claude-haiku-4-5-20251001",
|
||||
"input_tokens": response.usage.input_tokens,
|
||||
"output_tokens": response.usage.output_tokens,
|
||||
"total_tokens": response.usage.input_tokens + response.usage.output_tokens,
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
except json.JSONDecodeError:
|
||||
print("::error::Failed to parse JSON from Claude response", file=sys.stderr)
|
||||
print(f"Response text: {response_text[:500]}", file=sys.stderr)
|
||||
return {
|
||||
"functions_documented": [],
|
||||
"summary": {"total_documented": 0},
|
||||
"_metadata": {"error": "JSON parse error"},
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"::error::API error: {e}", file=sys.stderr)
|
||||
return {
|
||||
"functions_documented": [],
|
||||
"summary": {"total_documented": 0},
|
||||
"_metadata": {"error": str(e)},
|
||||
}
|
||||
|
||||
|
||||
def estimate_cost(input_tokens: int, output_tokens: int) -> float:
|
||||
"""Estimate cost based on Claude Haiku 4.5 pricing."""
|
||||
input_cost = (input_tokens / 1_000_000) * 1
|
||||
output_cost = (output_tokens / 1_000_000) * 5
|
||||
return round(input_cost + output_cost, 2)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Main entry point for docstring generation."""
|
||||
docs_path = Path(sys.argv[1] if len(sys.argv) > 1 else "/tmp/missing-docs.json")
|
||||
output_path = Path(
|
||||
sys.argv[2] if len(sys.argv) > 2 else "/tmp/documentation-suggestions.json"
|
||||
)
|
||||
|
||||
print(f"Loading functions without docstrings from {docs_path}...")
|
||||
functions = load_missing_docs(docs_path)
|
||||
|
||||
if not functions:
|
||||
print("No functions need docstrings. Skipping generation.")
|
||||
result = {
|
||||
"functions_documented": [],
|
||||
"summary": {"total_documented": 0},
|
||||
"_metadata": {
|
||||
"model": "N/A",
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
},
|
||||
}
|
||||
else:
|
||||
print(f"Generating docstrings for {len(functions)} functions...")
|
||||
|
||||
all_results = []
|
||||
total_input_tokens = 0
|
||||
total_output_tokens = 0
|
||||
|
||||
batch_size = 8
|
||||
for i in range(0, len(functions), batch_size):
|
||||
batch = functions[i : i + batch_size]
|
||||
print(f"Processing batch {i // batch_size + 1} ({len(batch)} functions)...")
|
||||
|
||||
batch_result = generate_docstrings_batch(batch)
|
||||
all_results.extend(batch_result.get("functions_documented", []))
|
||||
|
||||
metadata = batch_result.get("_metadata", {})
|
||||
total_input_tokens += metadata.get("input_tokens", 0)
|
||||
total_output_tokens += metadata.get("output_tokens", 0)
|
||||
|
||||
result = {
|
||||
"functions_documented": all_results,
|
||||
"summary": {"total_documented": len(all_results)},
|
||||
"_metadata": {
|
||||
"model": "claude-haiku-4-5-20251001",
|
||||
"input_tokens": total_input_tokens,
|
||||
"output_tokens": total_output_tokens,
|
||||
"total_tokens": total_input_tokens + total_output_tokens,
|
||||
},
|
||||
}
|
||||
|
||||
with open(output_path, "w") as f:
|
||||
json.dump(result, f, indent=2)
|
||||
|
||||
print(f"Docstring generation complete. Results saved to {output_path}")
|
||||
|
||||
summary = result.get("summary", {})
|
||||
metadata = result.get("_metadata", {})
|
||||
|
||||
print("\n## Documentation Generation Results\n")
|
||||
print(f"- **Functions documented**: {summary.get('total_documented', 0)}")
|
||||
|
||||
if metadata.get("input_tokens"):
|
||||
cost = estimate_cost(metadata["input_tokens"], metadata["output_tokens"])
|
||||
print(f"\n**Cost**: ${cost}")
|
||||
print(
|
||||
f"**Tokens**: {metadata['total_tokens']:,} ({metadata['input_tokens']:,} in + {metadata['output_tokens']:,} out)"
|
||||
)
|
||||
|
||||
github_output = os.environ.get("GITHUB_OUTPUT", "")
|
||||
if github_output:
|
||||
with open(github_output, "a") as f:
|
||||
f.write(f"docstring_cost={cost}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Type Hints Generator using Claude Haiku 4.5
|
||||
|
||||
Generates type hints for functions missing annotations using Claude Haiku 4.5.
|
||||
Processes in batches of 10 functions per API call for cost efficiency.
|
||||
|
||||
Cost: $27-47 per run (depending on function count and complexity)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from anthropic_helper import create_message
|
||||
|
||||
|
||||
def load_missing_hints(hints_path: Path) -> list[dict[str, Any]]:
|
||||
"""Load functions without type hints from JSON."""
|
||||
try:
|
||||
with open(hints_path, "r") as f:
|
||||
hints = json.load(f)
|
||||
return hints
|
||||
except FileNotFoundError:
|
||||
print(f"::error::Hints file not found: {hints_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except json.JSONDecodeError:
|
||||
print(f"::error::Invalid JSON in hints file: {hints_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def read_function_code(file_path: str, function_name: str, line: int) -> str:
|
||||
"""Read the function code from file."""
|
||||
try:
|
||||
with open(file_path, "r") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
start_line = line - 1
|
||||
code_lines = []
|
||||
indent_level = None
|
||||
|
||||
for i in range(start_line, len(lines)):
|
||||
line_text = lines[i]
|
||||
|
||||
if indent_level is None:
|
||||
indent_level = len(line_text) - len(line_text.lstrip())
|
||||
|
||||
current_indent = len(line_text) - len(line_text.lstrip())
|
||||
if i > start_line and current_indent <= indent_level and line_text.strip():
|
||||
if line_text.strip().startswith(("def ", "class ", "@")):
|
||||
break
|
||||
|
||||
code_lines.append(line_text)
|
||||
|
||||
if len(code_lines) >= 50:
|
||||
break
|
||||
|
||||
return "".join(code_lines)
|
||||
except Exception as e:
|
||||
return f"# Could not read function code: {e}"
|
||||
|
||||
|
||||
def build_type_hints_prompt(functions: list[dict[str, Any]]) -> str:
|
||||
"""Build the prompt for generating type hints."""
|
||||
functions_formatted = []
|
||||
|
||||
for i, func in enumerate(functions, 1):
|
||||
code = read_function_code(func["file"], func["function"], func["line"])
|
||||
functions_formatted.append(f"""
|
||||
### Function {i}: `{func["function"]}` in `{func["file"]}`
|
||||
|
||||
```python
|
||||
{code}
|
||||
```
|
||||
""")
|
||||
|
||||
functions_text = "\n".join(functions_formatted)
|
||||
|
||||
return f"""You are a Python type hints expert. Generate accurate type hints for functions missing annotations.
|
||||
|
||||
## Functions to Annotate ({len(functions)} total)
|
||||
|
||||
{functions_text}
|
||||
|
||||
## Your Task
|
||||
|
||||
For each function, provide:
|
||||
1. **Typed function signature** with parameter and return type annotations
|
||||
2. **Import statements** needed for the types
|
||||
3. **Justification** explaining your type choices
|
||||
|
||||
## Guidelines
|
||||
|
||||
- Use standard library types when possible (`list`, `dict`, `str`, `int`, etc.)
|
||||
- Use `typing` module for complex types (`List[str]`, `Optional[int]`, `Union`, etc.)
|
||||
- Use `Any` sparingly - only when truly dynamic
|
||||
- For async functions, annotate return type as `Awaitable[T]` or use `async def`
|
||||
- Consider None returns: use `Optional[T]` if function can return None
|
||||
- Look at parameter usage in function body to infer types
|
||||
- Check existing return statements for return type hints
|
||||
|
||||
## Output Format
|
||||
|
||||
Respond with JSON:
|
||||
|
||||
```json
|
||||
{{
|
||||
"functions_annotated": [
|
||||
{{
|
||||
"file": "app/example.py",
|
||||
"function": "process_data",
|
||||
"line": 42,
|
||||
"original_signature": "def process_data(data, options=None):",
|
||||
"typed_signature": "def process_data(data: List[dict], options: Optional[dict] = None) -> dict:",
|
||||
"imports_needed": ["from typing import List, Optional"],
|
||||
"justification": "data is iterated as list of dicts, options checked for None, returns dict"
|
||||
}}
|
||||
],
|
||||
"summary": {{
|
||||
"total_annotated": 5,
|
||||
"imports_added": ["typing.List", "typing.Optional", "typing.Union"]
|
||||
}}
|
||||
}}
|
||||
```
|
||||
|
||||
Begin your analysis."""
|
||||
|
||||
|
||||
def generate_type_hints_batch(
|
||||
functions: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
"""Generate type hints for a batch of functions using Claude Haiku 4.5."""
|
||||
prompt = build_type_hints_prompt(functions)
|
||||
|
||||
try:
|
||||
response = create_message(
|
||||
model="claude-haiku-4-5-20251001",
|
||||
max_tokens=4096,
|
||||
temperature=0,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
)
|
||||
|
||||
response_text = ""
|
||||
for block in response.content:
|
||||
if block.type == "text":
|
||||
response_text += block.text
|
||||
|
||||
json_start = response_text.find("```json")
|
||||
if json_start != -1:
|
||||
json_start = response_text.find("\n", json_start) + 1
|
||||
json_end = response_text.find("```", json_start)
|
||||
response_text = response_text[json_start:json_end].strip()
|
||||
else:
|
||||
json_start = response_text.find("{")
|
||||
json_end = response_text.rfind("}") + 1
|
||||
if json_start != -1 and json_end > json_start:
|
||||
response_text = response_text[json_start:json_end]
|
||||
|
||||
result = json.loads(response_text)
|
||||
|
||||
result["_metadata"] = {
|
||||
"model": "claude-haiku-4-5-20251001",
|
||||
"input_tokens": response.usage.input_tokens,
|
||||
"output_tokens": response.usage.output_tokens,
|
||||
"total_tokens": response.usage.input_tokens + response.usage.output_tokens,
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
except json.JSONDecodeError:
|
||||
print("::error::Failed to parse JSON from Claude response", file=sys.stderr)
|
||||
print(f"Response text: {response_text[:500]}", file=sys.stderr)
|
||||
return {
|
||||
"functions_annotated": [],
|
||||
"summary": {"total_annotated": 0, "imports_added": []},
|
||||
"_metadata": {"error": "JSON parse error"},
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"::error::API error: {e}", file=sys.stderr)
|
||||
return {
|
||||
"functions_annotated": [],
|
||||
"summary": {"total_annotated": 0, "imports_added": []},
|
||||
"_metadata": {"error": str(e)},
|
||||
}
|
||||
|
||||
|
||||
def estimate_cost(input_tokens: int, output_tokens: int) -> float:
|
||||
"""Estimate cost based on Claude Haiku 4.5 pricing."""
|
||||
input_cost = (input_tokens / 1_000_000) * 1
|
||||
output_cost = (output_tokens / 1_000_000) * 5
|
||||
return round(input_cost + output_cost, 2)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Main entry point for type hints generation."""
|
||||
hints_path = Path(sys.argv[1] if len(sys.argv) > 1 else "/tmp/missing-hints.json")
|
||||
output_path = Path(
|
||||
sys.argv[2] if len(sys.argv) > 2 else "/tmp/type-hints-suggestions.json"
|
||||
)
|
||||
|
||||
print(f"Loading functions without type hints from {hints_path}...")
|
||||
functions = load_missing_hints(hints_path)
|
||||
|
||||
if not functions:
|
||||
print("No functions need type hints. Skipping generation.")
|
||||
result = {
|
||||
"functions_annotated": [],
|
||||
"summary": {"total_annotated": 0, "imports_added": []},
|
||||
"_metadata": {
|
||||
"model": "N/A",
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
},
|
||||
}
|
||||
else:
|
||||
print(f"Generating type hints for {len(functions)} functions...")
|
||||
|
||||
all_results = []
|
||||
total_input_tokens = 0
|
||||
total_output_tokens = 0
|
||||
|
||||
batch_size = 10
|
||||
for i in range(0, len(functions), batch_size):
|
||||
batch = functions[i : i + batch_size]
|
||||
print(f"Processing batch {i // batch_size + 1} ({len(batch)} functions)...")
|
||||
|
||||
batch_result = generate_type_hints_batch(batch)
|
||||
all_results.extend(batch_result.get("functions_annotated", []))
|
||||
|
||||
metadata = batch_result.get("_metadata", {})
|
||||
total_input_tokens += metadata.get("input_tokens", 0)
|
||||
total_output_tokens += metadata.get("output_tokens", 0)
|
||||
|
||||
all_imports = set()
|
||||
for func in all_results:
|
||||
all_imports.update(func.get("imports_needed", []))
|
||||
|
||||
result = {
|
||||
"functions_annotated": all_results,
|
||||
"summary": {
|
||||
"total_annotated": len(all_results),
|
||||
"imports_added": sorted(all_imports),
|
||||
},
|
||||
"_metadata": {
|
||||
"model": "claude-haiku-4-5-20251001",
|
||||
"input_tokens": total_input_tokens,
|
||||
"output_tokens": total_output_tokens,
|
||||
"total_tokens": total_input_tokens + total_output_tokens,
|
||||
},
|
||||
}
|
||||
|
||||
with open(output_path, "w") as f:
|
||||
json.dump(result, f, indent=2)
|
||||
|
||||
print(f"Type hints generation complete. Results saved to {output_path}")
|
||||
|
||||
summary = result.get("summary", {})
|
||||
metadata = result.get("_metadata", {})
|
||||
|
||||
print("\n## Type Hints Generation Results\n")
|
||||
print(f"- **Functions annotated**: {summary.get('total_annotated', 0)}")
|
||||
print(f"- **Imports needed**: {len(summary.get('imports_added', []))}")
|
||||
|
||||
if metadata.get("input_tokens"):
|
||||
cost = estimate_cost(metadata["input_tokens"], metadata["output_tokens"])
|
||||
print(f"\n**Cost**: ${cost}")
|
||||
print(
|
||||
f"**Tokens**: {metadata['total_tokens']:,} ({metadata['input_tokens']:,} in + {metadata['output_tokens']:,} out)"
|
||||
)
|
||||
|
||||
github_output = os.environ.get("GITHUB_OUTPUT", "")
|
||||
if github_output:
|
||||
with open(github_output, "a") as f:
|
||||
f.write(f"type_hints_cost={cost}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user