chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,398 @@
|
||||
"""Analyze git diffs to determine which directories need to be tested.
|
||||
|
||||
Intelligently determines which LangChain packages and directories need to be tested,
|
||||
linted, or built based on the changes. Handles dependency relationships between
|
||||
packages, maps file changes to appropriate CI job configurations, and outputs JSON
|
||||
configurations for GitHub Actions.
|
||||
|
||||
- Maps changed files to affected package directories (libs/core, libs/partners/*, etc.)
|
||||
- Builds dependency graph to include dependent packages when core components change
|
||||
- Generates test matrix configurations with appropriate Python versions
|
||||
- Handles special cases for Pydantic version testing and performance benchmarks
|
||||
|
||||
Used as part of the check_diffs workflow.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Set
|
||||
|
||||
import tomllib
|
||||
from get_min_versions import get_min_version_from_toml
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
LANGCHAIN_DIRS = [
|
||||
"libs/core",
|
||||
"libs/text-splitters",
|
||||
"libs/langchain",
|
||||
"libs/langchain_v1",
|
||||
"libs/model-profiles",
|
||||
]
|
||||
|
||||
# Packages with VCR cassette-backed integration tests.
|
||||
# These get a playback-only CI check to catch stale cassettes.
|
||||
VCR_PACKAGES = {
|
||||
"libs/partners/openai",
|
||||
}
|
||||
|
||||
# When set to True, we are ignoring core dependents
|
||||
# in order to be able to get CI to pass for each individual
|
||||
# package that depends on core
|
||||
# e.g. if you touch core, we don't then add textsplitters/etc to CI
|
||||
IGNORE_CORE_DEPENDENTS = False
|
||||
|
||||
# Ignored partners are removed from dependents but still run if directly edited
|
||||
IGNORED_PARTNERS = [
|
||||
# remove huggingface from dependents because of CI instability
|
||||
# specifically in huggingface jobs
|
||||
"huggingface",
|
||||
]
|
||||
|
||||
|
||||
def all_package_dirs() -> Set[str]:
|
||||
return {
|
||||
"/".join(path.split("/")[:-1]).lstrip("./")
|
||||
for path in glob.glob("./libs/**/pyproject.toml", recursive=True)
|
||||
if "libs/standard-tests" not in path
|
||||
}
|
||||
|
||||
|
||||
def dependents_graph() -> dict:
|
||||
"""Construct a mapping of package -> dependents
|
||||
|
||||
Done such that we can run tests on all dependents of a package when a change is made.
|
||||
"""
|
||||
dependents = defaultdict(set)
|
||||
|
||||
for path in glob.glob("./libs/**/pyproject.toml", recursive=True):
|
||||
if "template" in path:
|
||||
continue
|
||||
|
||||
# load regular and test deps from pyproject.toml
|
||||
with open(path, "rb") as f:
|
||||
pyproject = tomllib.load(f)
|
||||
|
||||
pkg_dir = "libs" + "/".join(path.split("libs")[1].split("/")[:-1])
|
||||
for dep in [
|
||||
*pyproject["project"]["dependencies"],
|
||||
*pyproject["dependency-groups"]["test"],
|
||||
]:
|
||||
requirement = Requirement(dep)
|
||||
package_name = requirement.name
|
||||
if "langchain" in dep:
|
||||
dependents[package_name].add(pkg_dir)
|
||||
continue
|
||||
|
||||
# load extended deps from extended_testing_deps.txt
|
||||
package_path = Path(path).parent
|
||||
extended_requirement_path = package_path / "extended_testing_deps.txt"
|
||||
if extended_requirement_path.exists():
|
||||
with open(extended_requirement_path, "r") as f:
|
||||
extended_deps = f.read().splitlines()
|
||||
for depline in extended_deps:
|
||||
if depline.startswith("-e "):
|
||||
# editable dependency
|
||||
assert depline.startswith("-e ../partners/"), (
|
||||
"Extended test deps should only editable install partner packages"
|
||||
)
|
||||
partner = depline.split("partners/")[1]
|
||||
dep = f"langchain-{partner}"
|
||||
else:
|
||||
dep = depline.split("==")[0]
|
||||
|
||||
if "langchain" in dep:
|
||||
dependents[dep].add(pkg_dir)
|
||||
|
||||
for k in dependents:
|
||||
for partner in IGNORED_PARTNERS:
|
||||
if f"libs/partners/{partner}" in dependents[k]:
|
||||
dependents[k].remove(f"libs/partners/{partner}")
|
||||
return dependents
|
||||
|
||||
|
||||
def add_dependents(dirs_to_eval: Set[str], dependents: dict) -> List[str]:
|
||||
updated = set()
|
||||
for dir_ in dirs_to_eval:
|
||||
# handle core manually because it has so many dependents
|
||||
if "core" in dir_:
|
||||
updated.add(dir_)
|
||||
continue
|
||||
pkg = "langchain-" + dir_.split("/")[-1]
|
||||
updated.update(dependents[pkg])
|
||||
updated.add(dir_)
|
||||
return list(updated)
|
||||
|
||||
|
||||
def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]:
|
||||
if job == "test-pydantic":
|
||||
return _get_pydantic_test_configs(dir_)
|
||||
|
||||
if job == "codspeed":
|
||||
# CPU simulation (<1% variance, Valgrind-based) is the default.
|
||||
# Partners with heavy SDK inits use walltime instead to keep CI fast.
|
||||
CODSPEED_WALLTIME_DIRS = {
|
||||
"libs/core",
|
||||
"libs/partners/fireworks", # ~328s under simulation
|
||||
"libs/partners/openai", # 6 benchmarks, ~6 min under simulation
|
||||
}
|
||||
mode = "walltime" if dir_ in CODSPEED_WALLTIME_DIRS else "simulation"
|
||||
return [
|
||||
{
|
||||
"working-directory": dir_,
|
||||
"python-version": "3.13",
|
||||
"codspeed-mode": mode,
|
||||
}
|
||||
]
|
||||
if dir_ == "libs/core":
|
||||
py_versions = ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
else:
|
||||
py_versions = ["3.10", "3.14"]
|
||||
|
||||
return [{"working-directory": dir_, "python-version": py_v} for py_v in py_versions]
|
||||
|
||||
|
||||
def _get_pydantic_test_configs(
|
||||
dir_: str, *, python_version: str = "3.12"
|
||||
) -> List[Dict[str, str]]:
|
||||
with open("./libs/core/uv.lock", "rb") as f:
|
||||
core_uv_lock_data = tomllib.load(f)
|
||||
for package in core_uv_lock_data["package"]:
|
||||
if package["name"] == "pydantic":
|
||||
core_max_pydantic_minor = package["version"].split(".")[1]
|
||||
break
|
||||
|
||||
with open(f"./{dir_}/uv.lock", "rb") as f:
|
||||
dir_uv_lock_data = tomllib.load(f)
|
||||
|
||||
for package in dir_uv_lock_data["package"]:
|
||||
if package["name"] == "pydantic":
|
||||
dir_max_pydantic_minor = package["version"].split(".")[1]
|
||||
break
|
||||
|
||||
core_min_pydantic_version = get_min_version_from_toml(
|
||||
"./libs/core/pyproject.toml", "release", python_version, include=["pydantic"]
|
||||
)["pydantic"]
|
||||
core_min_pydantic_minor = (
|
||||
core_min_pydantic_version.split(".")[1]
|
||||
if "." in core_min_pydantic_version
|
||||
else "0"
|
||||
)
|
||||
dir_min_pydantic_version = get_min_version_from_toml(
|
||||
f"./{dir_}/pyproject.toml", "release", python_version, include=["pydantic"]
|
||||
).get("pydantic", "0.0.0")
|
||||
dir_min_pydantic_minor = (
|
||||
dir_min_pydantic_version.split(".")[1]
|
||||
if "." in dir_min_pydantic_version
|
||||
else "0"
|
||||
)
|
||||
|
||||
max_pydantic_minor = min(
|
||||
int(dir_max_pydantic_minor),
|
||||
int(core_max_pydantic_minor),
|
||||
)
|
||||
min_pydantic_minor = max(
|
||||
int(dir_min_pydantic_minor),
|
||||
int(core_min_pydantic_minor),
|
||||
)
|
||||
|
||||
configs = [
|
||||
{
|
||||
"working-directory": dir_,
|
||||
"pydantic-version": f"2.{v}.0",
|
||||
"python-version": python_version,
|
||||
}
|
||||
for v in range(min_pydantic_minor, max_pydantic_minor + 1)
|
||||
]
|
||||
return configs
|
||||
|
||||
|
||||
def _get_configs_for_multi_dirs(
|
||||
job: str, dirs_to_run: Dict[str, Set[str]], dependents: dict
|
||||
) -> List[Dict[str, str]]:
|
||||
if job == "lint":
|
||||
dirs = add_dependents(
|
||||
dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"],
|
||||
dependents,
|
||||
)
|
||||
elif job in ["test", "compile-integration-tests", "dependencies", "test-pydantic"]:
|
||||
dirs = add_dependents(
|
||||
dirs_to_run["test"] | dirs_to_run["extended-test"], dependents
|
||||
)
|
||||
elif job == "extended-tests":
|
||||
dirs = list(dirs_to_run["extended-test"])
|
||||
elif job == "codspeed":
|
||||
dirs = list(dirs_to_run["codspeed"])
|
||||
elif job == "vcr-tests":
|
||||
# Only run VCR tests for packages that have cassettes and are affected
|
||||
all_affected = set(
|
||||
add_dependents(
|
||||
dirs_to_run["test"] | dirs_to_run["extended-test"], dependents
|
||||
)
|
||||
)
|
||||
dirs = [d for d in VCR_PACKAGES if d in all_affected]
|
||||
else:
|
||||
raise ValueError(f"Unknown job: {job}")
|
||||
|
||||
return [
|
||||
config for dir_ in dirs for config in _get_configs_for_single_dir(job, dir_)
|
||||
]
|
||||
|
||||
|
||||
def _get_changed_files(args: list[str]) -> list[str]:
|
||||
"""Parse changed files from command-line arguments.
|
||||
|
||||
Args:
|
||||
args: Either a legacy list of filename arguments or a single JSON array
|
||||
produced by `Ana06/get-changed-files` with `format: json`.
|
||||
|
||||
Returns:
|
||||
List of changed files.
|
||||
|
||||
Raises:
|
||||
ValueError: If a single argument looks like JSON but is not a string array.
|
||||
"""
|
||||
if len(args) != 1:
|
||||
return args
|
||||
|
||||
value = args[0].strip()
|
||||
if not value.startswith("[") or not value.endswith("]"):
|
||||
return args
|
||||
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
msg = "Expected changed files JSON to be a list of strings."
|
||||
raise ValueError(msg) from e
|
||||
|
||||
if not isinstance(parsed, list) or not all(
|
||||
isinstance(file, str) for file in parsed
|
||||
):
|
||||
msg = "Expected changed files JSON to be a list of strings."
|
||||
raise ValueError(msg)
|
||||
return parsed
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
files = _get_changed_files(sys.argv[1:])
|
||||
|
||||
dirs_to_run: Dict[str, set] = {
|
||||
"lint": set(),
|
||||
"test": set(),
|
||||
"extended-test": set(),
|
||||
"codspeed": set(),
|
||||
}
|
||||
docs_edited = False
|
||||
|
||||
if len(files) >= 300:
|
||||
# max diff length is 300 files - there are likely files missing
|
||||
dirs_to_run["lint"] = all_package_dirs()
|
||||
dirs_to_run["test"] = all_package_dirs()
|
||||
dirs_to_run["extended-test"] = set(LANGCHAIN_DIRS)
|
||||
|
||||
for file in files:
|
||||
if any(
|
||||
file.startswith(dir_)
|
||||
for dir_ in (
|
||||
".github/workflows",
|
||||
".github/tools",
|
||||
".github/actions",
|
||||
".github/scripts/check_diff.py",
|
||||
)
|
||||
):
|
||||
# Infrastructure changes (workflows, actions, CI scripts) trigger tests on
|
||||
# all core packages as a safety measure. This ensures that changes to CI/CD
|
||||
# infrastructure don't inadvertently break package testing, even if the change
|
||||
# appears unrelated (e.g., documentation build workflows). This is intentionally
|
||||
# conservative to catch unexpected side effects from workflow modifications.
|
||||
#
|
||||
# Example: A PR modifying .github/workflows/api_doc_build.yml will trigger
|
||||
# lint/test jobs for libs/core, libs/text-splitters, libs/langchain, and
|
||||
# libs/langchain_v1, even though the workflow may only affect documentation.
|
||||
dirs_to_run["extended-test"].update(LANGCHAIN_DIRS)
|
||||
|
||||
if file.startswith("libs/core"):
|
||||
dirs_to_run["codspeed"].add("libs/core")
|
||||
if file.startswith("libs/langchain_v1"):
|
||||
dirs_to_run["codspeed"].add("libs/langchain_v1")
|
||||
if any(file.startswith(dir_) for dir_ in LANGCHAIN_DIRS):
|
||||
# add that dir and all dirs after in LANGCHAIN_DIRS
|
||||
# for extended testing
|
||||
|
||||
found = False
|
||||
for dir_ in LANGCHAIN_DIRS:
|
||||
if dir_ == "libs/core" and IGNORE_CORE_DEPENDENTS:
|
||||
dirs_to_run["extended-test"].add(dir_)
|
||||
continue
|
||||
if file.startswith(dir_):
|
||||
found = True
|
||||
if found:
|
||||
dirs_to_run["extended-test"].add(dir_)
|
||||
elif file.startswith("libs/standard-tests"):
|
||||
# TODO: update to include all packages that rely on standard-tests (all partner packages)
|
||||
# Note: won't run on external repo partners
|
||||
dirs_to_run["lint"].add("libs/standard-tests")
|
||||
dirs_to_run["test"].add("libs/standard-tests")
|
||||
dirs_to_run["test"].add("libs/partners/mistralai")
|
||||
dirs_to_run["test"].add("libs/partners/openai")
|
||||
dirs_to_run["test"].add("libs/partners/anthropic")
|
||||
dirs_to_run["test"].add("libs/partners/fireworks")
|
||||
dirs_to_run["test"].add("libs/partners/groq")
|
||||
|
||||
elif file.startswith("libs/partners"):
|
||||
partner_dir = file.split("/")[2]
|
||||
if os.path.isdir(f"libs/partners/{partner_dir}") and [
|
||||
filename
|
||||
for filename in os.listdir(f"libs/partners/{partner_dir}")
|
||||
if not filename.startswith(".")
|
||||
] != ["README.md"]:
|
||||
dirs_to_run["test"].add(f"libs/partners/{partner_dir}")
|
||||
# Only add to codspeed if the partner has benchmarks and is not ignored
|
||||
if (
|
||||
partner_dir not in IGNORED_PARTNERS
|
||||
and os.path.isdir(
|
||||
f"libs/partners/{partner_dir}/tests/benchmarks"
|
||||
)
|
||||
):
|
||||
dirs_to_run["codspeed"].add(f"libs/partners/{partner_dir}")
|
||||
# Skip if the directory was deleted or is just a tombstone readme
|
||||
elif file.startswith("libs/"):
|
||||
# Check if this is a root-level file in libs/ (e.g., libs/README.md)
|
||||
file_parts = file.split("/")
|
||||
if len(file_parts) == 2:
|
||||
# Root-level file in libs/, skip it (no tests needed)
|
||||
continue
|
||||
raise ValueError(
|
||||
f"Unknown lib: {file}. check_diff.py likely needs "
|
||||
"an update for this new library!"
|
||||
)
|
||||
elif file in [
|
||||
"pyproject.toml",
|
||||
"uv.lock",
|
||||
]: # root uv files
|
||||
docs_edited = True
|
||||
|
||||
dependents = dependents_graph()
|
||||
|
||||
# we now have dirs_by_job
|
||||
# todo: clean this up
|
||||
map_job_to_configs = {
|
||||
job: _get_configs_for_multi_dirs(job, dirs_to_run, dependents)
|
||||
for job in [
|
||||
"lint",
|
||||
"test",
|
||||
"extended-tests",
|
||||
"compile-integration-tests",
|
||||
"dependencies",
|
||||
"test-pydantic",
|
||||
"codspeed",
|
||||
"vcr-tests",
|
||||
]
|
||||
}
|
||||
|
||||
for key, value in map_job_to_configs.items():
|
||||
json_output = json.dumps(value)
|
||||
print(f"{key}={json_output}")
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Check that optional extras stay in sync with required dependencies.
|
||||
|
||||
When a package appears in both [project.dependencies] and
|
||||
[project.optional-dependencies], we ensure their version constraints match.
|
||||
This prevents silent version drift (e.g. bumping a required dep but
|
||||
forgetting the corresponding extra).
|
||||
"""
|
||||
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from re import compile as re_compile
|
||||
|
||||
# Matches the package name at the start of a PEP 508 dependency string.
|
||||
# Stops at the first non-name character; downstream code is responsible for
|
||||
# stripping extras (`[...]`) and env markers (`; ...`) from the remainder.
|
||||
_NAME_RE = re_compile(r"^([A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?)")
|
||||
|
||||
|
||||
def _normalize(name: str) -> str:
|
||||
"""Normalize a package name for equality comparison.
|
||||
|
||||
Lowercases and maps `-` and `.` to `_`. Looser than PEP 503
|
||||
(which uses `-` and collapses runs), but sufficient for matching the
|
||||
same package across two PEP 508 strings.
|
||||
|
||||
Returns:
|
||||
Lowercased, underscore-normalized package name.
|
||||
"""
|
||||
return name.lower().replace("-", "_").replace(".", "_")
|
||||
|
||||
|
||||
def _parse_dep(dep: str) -> tuple[str, str]:
|
||||
"""Return `(normalized_name, version_spec)` from a PEP 508 string.
|
||||
|
||||
Strips extras (`pkg[async]`), environment markers (`; python_version ...`),
|
||||
URL specifiers (`pkg @ git+...`), and whitespace so the returned
|
||||
`version_spec` is directly comparable between a required and optional dep.
|
||||
|
||||
Returns:
|
||||
Tuple of normalized package name and bare version specifier.
|
||||
|
||||
Raises:
|
||||
ValueError: If the dependency string cannot be parsed.
|
||||
"""
|
||||
match = _NAME_RE.match(dep)
|
||||
if not match:
|
||||
msg = f"Cannot parse dependency: {dep!r}"
|
||||
raise ValueError(msg)
|
||||
name = match.group(1)
|
||||
rest = dep[match.end() :].strip()
|
||||
|
||||
if rest.startswith("["):
|
||||
close = rest.find("]")
|
||||
if close == -1:
|
||||
msg = f"Unclosed extras bracket in dependency: {dep!r}"
|
||||
raise ValueError(msg)
|
||||
rest = rest[close + 1 :].strip()
|
||||
|
||||
if ";" in rest:
|
||||
rest = rest.split(";", 1)[0].strip()
|
||||
|
||||
# URL specifiers have no comparable version; treat as unconstrained.
|
||||
if rest.startswith("@"):
|
||||
rest = ""
|
||||
|
||||
rest = " ".join(rest.split())
|
||||
return _normalize(name), rest
|
||||
|
||||
|
||||
def main(pyproject_path: Path) -> int:
|
||||
"""Check extras sync and return `0` on pass, `1` on mismatch or parse error."""
|
||||
with pyproject_path.open("rb") as f:
|
||||
data = tomllib.load(f)
|
||||
|
||||
required: dict[str, str] = {}
|
||||
for dep in data.get("project", {}).get("dependencies", []):
|
||||
try:
|
||||
name, spec = _parse_dep(dep)
|
||||
except ValueError as e:
|
||||
print(f"::error file={pyproject_path}::{e}")
|
||||
return 1
|
||||
required[name] = spec
|
||||
|
||||
optional = data.get("project", {}).get("optional-dependencies", {})
|
||||
if not optional:
|
||||
return 0
|
||||
|
||||
mismatches: list[str] = []
|
||||
for group, deps in optional.items():
|
||||
for dep in deps:
|
||||
try:
|
||||
name, spec = _parse_dep(dep)
|
||||
except ValueError as e:
|
||||
print(f"::error file={pyproject_path}::{e}")
|
||||
return 1
|
||||
if name in required and spec != required[name]:
|
||||
mismatches.append(
|
||||
f" [{group}] {name}: extra has '{spec}' "
|
||||
f"but required dep has '{required[name]}'"
|
||||
)
|
||||
|
||||
if mismatches:
|
||||
print(f"Extra / required dependency version mismatch in {pyproject_path}:")
|
||||
print("\n".join(mismatches))
|
||||
print(
|
||||
"\nUpdate the optional extras in [project.optional-dependencies] "
|
||||
"to match [project.dependencies]."
|
||||
)
|
||||
return 1
|
||||
|
||||
print(f"All extras in {pyproject_path} are in sync with required dependencies.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("pyproject.toml")
|
||||
raise SystemExit(main(path))
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Check that no dependencies allow prereleases unless we're releasing a prerelease."""
|
||||
|
||||
import sys
|
||||
|
||||
import tomllib
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Get the TOML file path from the command line argument
|
||||
toml_file = sys.argv[1]
|
||||
|
||||
with open(toml_file, "rb") as file:
|
||||
toml_data = tomllib.load(file)
|
||||
|
||||
# See if we're releasing an rc or dev version
|
||||
version = toml_data["project"]["version"]
|
||||
releasing_rc = "rc" in version or "dev" in version
|
||||
|
||||
# If not, iterate through dependencies and make sure none allow prereleases
|
||||
if not releasing_rc:
|
||||
dependencies = toml_data["project"]["dependencies"]
|
||||
for dep_version in dependencies:
|
||||
dep_version_string = (
|
||||
dep_version["version"] if isinstance(dep_version, dict) else dep_version
|
||||
)
|
||||
|
||||
if "rc" in dep_version_string:
|
||||
raise ValueError(
|
||||
f"Dependency {dep_version} has a prerelease version. Please remove this."
|
||||
)
|
||||
|
||||
if isinstance(dep_version, dict) and dep_version.get(
|
||||
"allow-prereleases", False
|
||||
):
|
||||
raise ValueError(
|
||||
f"Dependency {dep_version} has allow-prereleases set to true. Please remove this."
|
||||
)
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Get minimum versions of dependencies from a pyproject.toml file."""
|
||||
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
else:
|
||||
# For Python 3.10 and below, which doesnt have stdlib tomllib
|
||||
import tomli as tomllib
|
||||
|
||||
import re
|
||||
from typing import List
|
||||
|
||||
import requests
|
||||
from packaging.requirements import Requirement
|
||||
from packaging.specifiers import SpecifierSet
|
||||
from packaging.version import Version, parse
|
||||
|
||||
MIN_VERSION_LIBS = [
|
||||
"langchain-core",
|
||||
"langchain",
|
||||
"langchain-text-splitters",
|
||||
"numpy",
|
||||
"SQLAlchemy",
|
||||
]
|
||||
|
||||
# some libs only get checked on release because of simultaneous changes in
|
||||
# multiple libs
|
||||
SKIP_IF_PULL_REQUEST = [
|
||||
"langchain-core",
|
||||
"langchain-text-splitters",
|
||||
"langchain",
|
||||
]
|
||||
|
||||
|
||||
def get_pypi_versions(package_name: str) -> List[str]:
|
||||
"""Fetch all available versions for a package from PyPI.
|
||||
|
||||
Args:
|
||||
package_name: Name of the package
|
||||
|
||||
Returns:
|
||||
List of all available versions
|
||||
|
||||
Raises:
|
||||
requests.exceptions.RequestException: If PyPI API request fails
|
||||
KeyError: If package not found or response format unexpected
|
||||
"""
|
||||
pypi_url = f"https://pypi.org/pypi/{package_name}/json"
|
||||
response = requests.get(pypi_url, timeout=10.0)
|
||||
response.raise_for_status()
|
||||
return list(response.json()["releases"].keys())
|
||||
|
||||
|
||||
def get_minimum_version(package_name: str, spec_string: str) -> str | None:
|
||||
"""Find the minimum published version that satisfies the given constraints.
|
||||
|
||||
Args:
|
||||
package_name: Name of the package
|
||||
spec_string: Version specification string (e.g., ">=0.2.43,<0.4.0,!=0.3.0")
|
||||
|
||||
Returns:
|
||||
Minimum compatible version or None if no compatible version found
|
||||
"""
|
||||
# Rewrite occurrences of ^0.0.z to 0.0.z (can be anywhere in constraint string)
|
||||
spec_string = re.sub(r"\^0\.0\.(\d+)", r"0.0.\1", spec_string)
|
||||
# Rewrite occurrences of ^0.y.z to >=0.y.z,<0.y+1 (can be anywhere in constraint string)
|
||||
for y in range(1, 10):
|
||||
spec_string = re.sub(
|
||||
rf"\^0\.{y}\.(\d+)", rf">=0.{y}.\1,<0.{y + 1}", spec_string
|
||||
)
|
||||
# Rewrite occurrences of ^x.y.z to >=x.y.z,<x+1.0.0 (can be anywhere in constraint string)
|
||||
for x in range(1, 10):
|
||||
spec_string = re.sub(
|
||||
rf"\^{x}\.(\d+)\.(\d+)", rf">={x}.\1.\2,<{x + 1}", spec_string
|
||||
)
|
||||
|
||||
spec_set = SpecifierSet(spec_string)
|
||||
all_versions = get_pypi_versions(package_name)
|
||||
|
||||
valid_versions = []
|
||||
for version_str in all_versions:
|
||||
try:
|
||||
version = parse(version_str)
|
||||
if spec_set.contains(version):
|
||||
valid_versions.append(version)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return str(min(valid_versions)) if valid_versions else None
|
||||
|
||||
|
||||
def _check_python_version_from_requirement(
|
||||
requirement: Requirement, python_version: str
|
||||
) -> bool:
|
||||
if not requirement.marker:
|
||||
return True
|
||||
else:
|
||||
marker_str = str(requirement.marker)
|
||||
if "python_version" in marker_str or "python_full_version" in marker_str:
|
||||
python_version_str = "".join(
|
||||
char
|
||||
for char in marker_str
|
||||
if char.isdigit() or char in (".", "<", ">", "=", ",")
|
||||
)
|
||||
return check_python_version(python_version, python_version_str)
|
||||
return True
|
||||
|
||||
|
||||
def get_min_version_from_toml(
|
||||
toml_path: str,
|
||||
versions_for: str,
|
||||
python_version: str,
|
||||
*,
|
||||
include: list | None = None,
|
||||
):
|
||||
# Parse the TOML file
|
||||
with open(toml_path, "rb") as file:
|
||||
toml_data = tomllib.load(file)
|
||||
|
||||
dependencies = defaultdict(list)
|
||||
for dep in toml_data["project"]["dependencies"]:
|
||||
requirement = Requirement(dep)
|
||||
dependencies[requirement.name].append(requirement)
|
||||
|
||||
# Initialize a dictionary to store the minimum versions
|
||||
min_versions = {}
|
||||
|
||||
# Iterate over the libs in MIN_VERSION_LIBS
|
||||
for lib in set(MIN_VERSION_LIBS + (include or [])):
|
||||
if versions_for == "pull_request" and lib in SKIP_IF_PULL_REQUEST:
|
||||
# some libs only get checked on release because of simultaneous
|
||||
# changes in multiple libs
|
||||
continue
|
||||
# Check if the lib is present in the dependencies
|
||||
if lib in dependencies:
|
||||
if include and lib not in include:
|
||||
continue
|
||||
requirements = dependencies[lib]
|
||||
for requirement in requirements:
|
||||
if _check_python_version_from_requirement(requirement, python_version):
|
||||
version_string = str(requirement.specifier)
|
||||
break
|
||||
|
||||
# Use parse_version to get the minimum supported version from version_string
|
||||
min_version = get_minimum_version(lib, version_string)
|
||||
|
||||
# Store the minimum version in the min_versions dictionary
|
||||
min_versions[lib] = min_version
|
||||
|
||||
return min_versions
|
||||
|
||||
|
||||
def check_python_version(version_string, constraint_string):
|
||||
"""Check if the given Python version matches the given constraints.
|
||||
|
||||
Args:
|
||||
version_string: A string representing the Python version (e.g. "3.8.5").
|
||||
constraint_string: A string representing the package's Python version
|
||||
constraints (e.g. ">=3.6, <4.0").
|
||||
|
||||
Returns:
|
||||
True if the version matches the constraints
|
||||
"""
|
||||
|
||||
# Rewrite occurrences of ^0.0.z to 0.0.z (can be anywhere in constraint string)
|
||||
constraint_string = re.sub(r"\^0\.0\.(\d+)", r"0.0.\1", constraint_string)
|
||||
# Rewrite occurrences of ^0.y.z to >=0.y.z,<0.y+1.0 (can be anywhere in constraint string)
|
||||
for y in range(1, 10):
|
||||
constraint_string = re.sub(
|
||||
rf"\^0\.{y}\.(\d+)", rf">=0.{y}.\1,<0.{y + 1}.0", constraint_string
|
||||
)
|
||||
# Rewrite occurrences of ^x.y.z to >=x.y.z,<x+1.0.0 (can be anywhere in constraint string)
|
||||
for x in range(1, 10):
|
||||
constraint_string = re.sub(
|
||||
rf"\^{x}\.0\.(\d+)", rf">={x}.0.\1,<{x + 1}.0.0", constraint_string
|
||||
)
|
||||
|
||||
try:
|
||||
version = Version(version_string)
|
||||
constraints = SpecifierSet(constraint_string)
|
||||
return version in constraints
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Get the TOML file path from the command line argument
|
||||
toml_file = sys.argv[1]
|
||||
versions_for = sys.argv[2]
|
||||
python_version = sys.argv[3]
|
||||
assert versions_for in ["release", "pull_request"]
|
||||
|
||||
# Call the function to get the minimum versions
|
||||
min_versions = get_min_version_from_toml(toml_file, versions_for, python_version)
|
||||
|
||||
# A `None` value means no *published* version on PyPI satisfies the declared
|
||||
# constraint, e.g. a `release(...)` PR bumped a minimum pin to a version that
|
||||
# has not shipped yet. Emitting `pkg==None` would be passed verbatim to
|
||||
# `uv pip install` in the release workflow's minimum-version test step,
|
||||
# producing a cryptic install failure, so fail loudly here instead.
|
||||
unresolved = [lib for lib, version in min_versions.items() if version is None]
|
||||
if unresolved:
|
||||
print(
|
||||
"ERROR: no published version on PyPI satisfies the declared constraint "
|
||||
f"for: {', '.join(sorted(unresolved))}. A release likely pinned a "
|
||||
"dependency to a version that is not yet published. Release the "
|
||||
"dependency first, or relax the pin.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
print(" ".join([f"{lib}=={version}" for lib, version in min_versions.items()]))
|
||||
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"trustedThreshold": 5,
|
||||
"labelColor": "b76e79",
|
||||
"sizeThresholds": [
|
||||
{ "label": "size: XS", "max": 50 },
|
||||
{ "label": "size: S", "max": 200 },
|
||||
{ "label": "size: M", "max": 500 },
|
||||
{ "label": "size: L", "max": 1000 },
|
||||
{ "label": "size: XL" }
|
||||
],
|
||||
"excludedFiles": ["uv.lock"],
|
||||
"excludedPaths": ["docs/"],
|
||||
"typeToLabel": {
|
||||
"feat": "feature",
|
||||
"fix": "fix",
|
||||
"docs": "documentation",
|
||||
"style": "linting",
|
||||
"refactor": "refactor",
|
||||
"perf": "performance",
|
||||
"test": "tests",
|
||||
"build": "infra",
|
||||
"ci": "infra",
|
||||
"chore": "infra",
|
||||
"revert": "revert",
|
||||
"release": "release",
|
||||
"hotfix": "hotfix",
|
||||
"breaking": "breaking"
|
||||
},
|
||||
"scopeToLabel": {
|
||||
"core": "core",
|
||||
"langchain": "langchain",
|
||||
"langchain-classic": "langchain-classic",
|
||||
"model-profiles": "model-profiles",
|
||||
"standard-tests": "standard-tests",
|
||||
"text-splitters": "text-splitters",
|
||||
"anthropic": "anthropic",
|
||||
"chroma": "chroma",
|
||||
"deepseek": "deepseek",
|
||||
"exa": "exa",
|
||||
"fireworks": "fireworks",
|
||||
"groq": "groq",
|
||||
"huggingface": "huggingface",
|
||||
"mistralai": "mistralai",
|
||||
"nomic": "nomic",
|
||||
"ollama": "ollama",
|
||||
"openai": "openai",
|
||||
"openrouter": "openrouter",
|
||||
"perplexity": "perplexity",
|
||||
"qdrant": "qdrant",
|
||||
"xai": "xai",
|
||||
"deps": "dependencies",
|
||||
"docs": "documentation",
|
||||
"infra": "infra"
|
||||
},
|
||||
"fileRules": [
|
||||
{ "label": "core", "prefix": "libs/core/", "skipExcludedFiles": true },
|
||||
{ "label": "langchain-classic", "prefix": "libs/langchain/", "skipExcludedFiles": true },
|
||||
{ "label": "langchain", "prefix": "libs/langchain_v1/", "skipExcludedFiles": true },
|
||||
{ "label": "standard-tests", "prefix": "libs/standard-tests/", "skipExcludedFiles": true },
|
||||
{ "label": "model-profiles", "prefix": "libs/model-profiles/", "skipExcludedFiles": true },
|
||||
{ "label": "text-splitters", "prefix": "libs/text-splitters/", "skipExcludedFiles": true },
|
||||
{ "label": "integration", "prefix": "libs/partners/", "skipExcludedFiles": true },
|
||||
{ "label": "anthropic", "prefix": "libs/partners/anthropic/", "skipExcludedFiles": true },
|
||||
{ "label": "chroma", "prefix": "libs/partners/chroma/", "skipExcludedFiles": true },
|
||||
{ "label": "deepseek", "prefix": "libs/partners/deepseek/", "skipExcludedFiles": true },
|
||||
{ "label": "exa", "prefix": "libs/partners/exa/", "skipExcludedFiles": true },
|
||||
{ "label": "fireworks", "prefix": "libs/partners/fireworks/", "skipExcludedFiles": true },
|
||||
{ "label": "groq", "prefix": "libs/partners/groq/", "skipExcludedFiles": true },
|
||||
{ "label": "huggingface", "prefix": "libs/partners/huggingface/", "skipExcludedFiles": true },
|
||||
{ "label": "mistralai", "prefix": "libs/partners/mistralai/", "skipExcludedFiles": true },
|
||||
{ "label": "nomic", "prefix": "libs/partners/nomic/", "skipExcludedFiles": true },
|
||||
{ "label": "ollama", "prefix": "libs/partners/ollama/", "skipExcludedFiles": true },
|
||||
{ "label": "openai", "prefix": "libs/partners/openai/", "skipExcludedFiles": true },
|
||||
{ "label": "openrouter", "prefix": "libs/partners/openrouter/", "skipExcludedFiles": true },
|
||||
{ "label": "perplexity", "prefix": "libs/partners/perplexity/", "skipExcludedFiles": true },
|
||||
{ "label": "qdrant", "prefix": "libs/partners/qdrant/", "skipExcludedFiles": true },
|
||||
{ "label": "xai", "prefix": "libs/partners/xai/", "skipExcludedFiles": true },
|
||||
{ "label": "github_actions", "prefix": ".github/workflows/" },
|
||||
{ "label": "github_actions", "prefix": ".github/actions/" },
|
||||
{ "label": "dependencies", "suffix": "pyproject.toml" },
|
||||
{ "label": "dependencies", "exact": "uv.lock" },
|
||||
{ "label": "dependencies", "pattern": "(?:^|/)requirements[^/]*\\.txt$" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
// Shared helpers for pr_labeler.yml and tag-external-issues.yml.
|
||||
//
|
||||
// Usage from actions/github-script (requires actions/checkout first):
|
||||
// const { h } = require('./.github/scripts/pr-labeler.js').loadAndInit(github, owner, repo, core);
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
function loadConfig() {
|
||||
const configPath = path.join(__dirname, 'pr-labeler-config.json');
|
||||
let raw;
|
||||
try {
|
||||
raw = fs.readFileSync(configPath, 'utf8');
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to read ${configPath}: ${e.message}`);
|
||||
}
|
||||
let config;
|
||||
try {
|
||||
config = JSON.parse(raw);
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to parse pr-labeler-config.json: ${e.message}`);
|
||||
}
|
||||
const required = [
|
||||
'labelColor', 'sizeThresholds', 'fileRules',
|
||||
'typeToLabel', 'scopeToLabel', 'trustedThreshold',
|
||||
'excludedFiles', 'excludedPaths',
|
||||
];
|
||||
const missing = required.filter(k => !(k in config));
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`pr-labeler-config.json missing required keys: ${missing.join(', ')}`);
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
function init(github, owner, repo, config, core) {
|
||||
if (!core) {
|
||||
throw new Error('init() requires a `core` parameter (e.g., from actions/github-script)');
|
||||
}
|
||||
const {
|
||||
trustedThreshold,
|
||||
labelColor,
|
||||
sizeThresholds,
|
||||
scopeToLabel,
|
||||
typeToLabel,
|
||||
fileRules: fileRulesDef,
|
||||
excludedFiles,
|
||||
excludedPaths,
|
||||
} = config;
|
||||
|
||||
const sizeLabels = sizeThresholds.map(t => t.label);
|
||||
const allTypeLabels = [...new Set(Object.values(typeToLabel))];
|
||||
const tierLabels = ['new-contributor', 'trusted-contributor'];
|
||||
|
||||
// ── Label management ──────────────────────────────────────────────
|
||||
|
||||
async function ensureLabel(name, color = labelColor) {
|
||||
try {
|
||||
await github.rest.issues.getLabel({ owner, repo, name });
|
||||
} catch (e) {
|
||||
if (e.status !== 404) throw e;
|
||||
try {
|
||||
await github.rest.issues.createLabel({ owner, repo, name, color });
|
||||
} catch (createErr) {
|
||||
// 422 = label created by a concurrent run between our get and create
|
||||
if (createErr.status !== 422) throw createErr;
|
||||
core.info(`Label "${name}" creation returned 422 (likely already exists)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Size calculation ──────────────────────────────────────────────
|
||||
|
||||
function getSizeLabel(totalChanged) {
|
||||
for (const t of sizeThresholds) {
|
||||
if (t.max != null && totalChanged < t.max) return t.label;
|
||||
}
|
||||
// Last entry has no max — it's the catch-all
|
||||
return sizeThresholds[sizeThresholds.length - 1].label;
|
||||
}
|
||||
|
||||
function computeSize(files) {
|
||||
const excluded = new Set(excludedFiles);
|
||||
const totalChanged = files.reduce((sum, f) => {
|
||||
const p = f.filename ?? '';
|
||||
const base = p.split('/').pop();
|
||||
if (excluded.has(base)) return sum;
|
||||
for (const prefix of excludedPaths) {
|
||||
if (p.startsWith(prefix)) return sum;
|
||||
}
|
||||
return sum + (f.additions ?? 0) + (f.deletions ?? 0);
|
||||
}, 0);
|
||||
return { totalChanged, sizeLabel: getSizeLabel(totalChanged) };
|
||||
}
|
||||
|
||||
// ── File-based labels ─────────────────────────────────────────────
|
||||
|
||||
function buildFileRules() {
|
||||
return fileRulesDef.map((rule, i) => {
|
||||
let test;
|
||||
if (rule.prefix) test = p => p.startsWith(rule.prefix);
|
||||
else if (rule.suffix) test = p => p.endsWith(rule.suffix);
|
||||
else if (rule.exact) test = p => p === rule.exact;
|
||||
else if (rule.pattern) {
|
||||
const re = new RegExp(rule.pattern);
|
||||
test = p => re.test(p);
|
||||
} else {
|
||||
throw new Error(
|
||||
`fileRules[${i}] (label: "${rule.label}") has no recognized matcher ` +
|
||||
`(expected one of: prefix, suffix, exact, pattern)`
|
||||
);
|
||||
}
|
||||
return { label: rule.label, test, skipExcluded: !!rule.skipExcludedFiles };
|
||||
});
|
||||
}
|
||||
|
||||
function matchFileLabels(files, fileRules) {
|
||||
const rules = fileRules || buildFileRules();
|
||||
const excluded = new Set(excludedFiles);
|
||||
const labels = new Set();
|
||||
for (const rule of rules) {
|
||||
// skipExcluded: ignore files whose basename is in the top-level
|
||||
// "excludedFiles" list (e.g. uv.lock) so lockfile-only changes
|
||||
// don't trigger package labels.
|
||||
const candidates = rule.skipExcluded
|
||||
? files.filter(f => !excluded.has((f.filename ?? '').split('/').pop()))
|
||||
: files;
|
||||
if (candidates.some(f => rule.test(f.filename ?? ''))) {
|
||||
labels.add(rule.label);
|
||||
}
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
|
||||
// ── Title-based labels ────────────────────────────────────────────
|
||||
|
||||
function matchTitleLabels(title) {
|
||||
const labels = new Set();
|
||||
const m = (title ?? '').match(/^(\w+)(?:\(([^)]+)\))?(!)?:/);
|
||||
if (!m) return { labels, type: null, typeLabel: null, scopes: [], breaking: false };
|
||||
|
||||
const type = m[1].toLowerCase();
|
||||
const scopeStr = m[2] ?? '';
|
||||
const breaking = !!m[3];
|
||||
|
||||
const typeLabel = typeToLabel[type] || null;
|
||||
if (typeLabel) labels.add(typeLabel);
|
||||
if (breaking) labels.add('breaking');
|
||||
|
||||
const scopes = scopeStr.split(',').map(s => s.trim()).filter(Boolean);
|
||||
for (const scope of scopes) {
|
||||
const sl = scopeToLabel[scope];
|
||||
if (sl) labels.add(sl);
|
||||
}
|
||||
|
||||
return { labels, type, typeLabel, scopes, breaking };
|
||||
}
|
||||
|
||||
// ── Org membership ────────────────────────────────────────────────
|
||||
|
||||
async function checkMembership(author, userType) {
|
||||
if (userType === 'Bot') {
|
||||
console.log(`${author} is a Bot — treating as internal`);
|
||||
return { isExternal: false };
|
||||
}
|
||||
|
||||
try {
|
||||
const membership = await github.rest.orgs.getMembershipForUser({
|
||||
org: 'langchain-ai',
|
||||
username: author,
|
||||
});
|
||||
const isExternal = membership.data.state !== 'active';
|
||||
console.log(
|
||||
isExternal
|
||||
? `${author} has pending membership — treating as external`
|
||||
: `${author} is an active member of langchain-ai`,
|
||||
);
|
||||
return { isExternal };
|
||||
} catch (e) {
|
||||
if (e.status === 404) {
|
||||
console.log(`${author} is not a member of langchain-ai`);
|
||||
return { isExternal: true };
|
||||
}
|
||||
// Non-404 errors (rate limit, auth failure, server error) must not
|
||||
// silently default to external — rethrow to fail the step.
|
||||
throw new Error(
|
||||
`Membership check failed for ${author} (${e.status}): ${e.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Contributor analysis ──────────────────────────────────────────
|
||||
|
||||
async function getContributorInfo(contributorCache, author, userType) {
|
||||
if (contributorCache.has(author)) return contributorCache.get(author);
|
||||
|
||||
const { isExternal } = await checkMembership(author, userType);
|
||||
|
||||
let mergedCount = null;
|
||||
if (isExternal) {
|
||||
try {
|
||||
const result = await github.rest.search.issuesAndPullRequests({
|
||||
q: `repo:${owner}/${repo} is:pr is:merged author:"${author}"`,
|
||||
per_page: 1,
|
||||
});
|
||||
mergedCount = result?.data?.total_count ?? null;
|
||||
} catch (e) {
|
||||
if (e?.status !== 422) throw e;
|
||||
core.warning(`Search failed for ${author}; skipping tier.`);
|
||||
}
|
||||
}
|
||||
|
||||
const info = { isExternal, mergedCount };
|
||||
contributorCache.set(author, info);
|
||||
return info;
|
||||
}
|
||||
|
||||
// ── Tier label resolution ───────────────────────────────────────────
|
||||
|
||||
async function applyTierLabel(issueNumber, author, { skipNewContributor = false } = {}) {
|
||||
let mergedCount;
|
||||
try {
|
||||
const result = await github.rest.search.issuesAndPullRequests({
|
||||
q: `repo:${owner}/${repo} is:pr is:merged author:"${author}"`,
|
||||
per_page: 1,
|
||||
});
|
||||
mergedCount = result?.data?.total_count;
|
||||
} catch (error) {
|
||||
if (error?.status !== 422) throw error;
|
||||
core.warning(`Search failed for ${author}; skipping tier label.`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (mergedCount == null) {
|
||||
core.warning(`Search response missing total_count for ${author}; skipping tier label.`);
|
||||
return;
|
||||
}
|
||||
|
||||
let tierLabel = null;
|
||||
if (mergedCount >= trustedThreshold) tierLabel = 'trusted-contributor';
|
||||
else if (mergedCount === 0 && !skipNewContributor) tierLabel = 'new-contributor';
|
||||
|
||||
if (tierLabel) {
|
||||
await ensureLabel(tierLabel);
|
||||
await github.rest.issues.addLabels({
|
||||
owner, repo, issue_number: issueNumber, labels: [tierLabel],
|
||||
});
|
||||
console.log(`Applied '${tierLabel}' to #${issueNumber} (${mergedCount} merged PRs)`);
|
||||
} else {
|
||||
console.log(`No tier label for ${author} (${mergedCount} merged PRs)`);
|
||||
}
|
||||
|
||||
return tierLabel;
|
||||
}
|
||||
|
||||
return {
|
||||
ensureLabel,
|
||||
getSizeLabel,
|
||||
computeSize,
|
||||
buildFileRules,
|
||||
matchFileLabels,
|
||||
matchTitleLabels,
|
||||
allTypeLabels,
|
||||
checkMembership,
|
||||
getContributorInfo,
|
||||
applyTierLabel,
|
||||
sizeLabels,
|
||||
tierLabels,
|
||||
trustedThreshold,
|
||||
labelColor,
|
||||
};
|
||||
}
|
||||
|
||||
function loadAndInit(github, owner, repo, core) {
|
||||
const config = loadConfig();
|
||||
return { config, h: init(github, owner, repo, config, core) };
|
||||
}
|
||||
|
||||
module.exports = { loadConfig, init, loadAndInit };
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Verify _release.yml dropdown options match actual package directories.
|
||||
|
||||
Dropdown options are short names (e.g. `openai`, `core`). The workflow's
|
||||
`EFFECTIVE_WORKING_DIR` expression re-adds the `libs/` prefix for top-level
|
||||
packages and `libs/partners/` for everything else. This test reconstructs the
|
||||
full path for each short name and compares against packages on disk.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
# Keep in sync with the non-partner allowlist in `EFFECTIVE_WORKING_DIR`
|
||||
# in `.github/workflows/_release.yml`.
|
||||
TOP_LEVEL_PACKAGES = frozenset(
|
||||
{"core", "langchain", "langchain_v1", "text-splitters", "standard-tests", "model-profiles"}
|
||||
)
|
||||
|
||||
|
||||
def _get_release_options() -> list[str]:
|
||||
workflow = REPO_ROOT / ".github" / "workflows" / "_release.yml"
|
||||
with open(workflow) as f:
|
||||
data = yaml.safe_load(f)
|
||||
try:
|
||||
# PyYAML (YAML 1.1) parses the bare key `on` as boolean True
|
||||
return data[True]["workflow_dispatch"]["inputs"]["working-directory"]["options"]
|
||||
except (KeyError, TypeError) as e:
|
||||
msg = f"Could not find workflow_dispatch options in {workflow}: {e}"
|
||||
raise AssertionError(msg) from e
|
||||
|
||||
|
||||
def _expand_option(option: str) -> str:
|
||||
if option in TOP_LEVEL_PACKAGES:
|
||||
return f"libs/{option}"
|
||||
return f"libs/partners/{option}"
|
||||
|
||||
|
||||
def _get_package_dirs() -> set[str]:
|
||||
libs = REPO_ROOT / "libs"
|
||||
dirs: set[str] = set()
|
||||
# Top-level packages (libs/core, libs/langchain, etc.)
|
||||
for p in libs.iterdir():
|
||||
if p.is_dir() and (p / "pyproject.toml").exists():
|
||||
dirs.add(f"libs/{p.name}")
|
||||
# Partner packages (libs/partners/*)
|
||||
partners = libs / "partners"
|
||||
if partners.exists():
|
||||
for p in partners.iterdir():
|
||||
if p.is_dir() and (p / "pyproject.toml").exists():
|
||||
dirs.add(f"libs/partners/{p.name}")
|
||||
return dirs
|
||||
|
||||
|
||||
def test_release_options_match_packages() -> None:
|
||||
options = {_expand_option(o) for o in _get_release_options()}
|
||||
packages = _get_package_dirs()
|
||||
missing_from_dropdown = packages - options
|
||||
extra_in_dropdown = options - packages
|
||||
assert not missing_from_dropdown, (
|
||||
f"Packages on disk missing from _release.yml dropdown: {missing_from_dropdown}"
|
||||
)
|
||||
assert not extra_in_dropdown, (
|
||||
f"Dropdown options with no matching package directory: {extra_in_dropdown}"
|
||||
)
|
||||
Reference in New Issue
Block a user