wehub-resource-sync
227 lines
8.5 KiB
Python
Executable File
227 lines
8.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Byte-identity audit: compare every vendored license file against upstream.
|
|
|
|
Verdicts:
|
|
IDENTICAL byte-equal to the upstream default-branch license
|
|
IDENTICAL@PINNED byte-equal to the license at the manifest's pinned commit
|
|
FIRST-PARTY-OK byte-equal to the project root LICENSE
|
|
FIRST-PARTY-VAR first-party but text differs from root LICENSE (inspect)
|
|
DIFFERS no byte-equal upstream candidate found (inspect)
|
|
ERROR upstream fetch failed
|
|
"""
|
|
import base64
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
GRAMMARS = os.path.join(ROOT, "internal/cbm/vendored/grammars")
|
|
|
|
FIRST_PARTY = {"cobol", "form", "janet", "magma", "protobuf", "wolfram"}
|
|
FORKS = { # self-maintained forks: vendored LICENSE must match the original upstream
|
|
"cfml": "cfmleditor/tree-sitter-cfml",
|
|
"cfscript": "cfmleditor/tree-sitter-cfml",
|
|
"dotenv": "pnx/tree-sitter-dotenv",
|
|
"qml": "yuja/tree-sitter-qmljs",
|
|
}
|
|
SPECIAL_NOTICE = {
|
|
"assembly": "RETAINED-MIT (upstream RubixDev/tree-sitter-assembly deleted from GitHub)",
|
|
"pine": "PROVENANCE-NOTICE (upstream kvarenzn/tree-sitter-pine declares ISC, ships no license file)",
|
|
}
|
|
DISAGREEMENT = {
|
|
"jinja2": "dbt-labs/tree-sitter-jinja2",
|
|
"just": "casey/tree-sitter-just",
|
|
"move": "tzakian/tree-sitter-move",
|
|
"sshconfig": "ObserverOfTime/tree-sitter-ssh-config",
|
|
"zsh": "georgeharker/tree-sitter-zsh",
|
|
}
|
|
LIBS = {
|
|
"vendored/mimalloc": ("microsoft/mimalloc", None),
|
|
"vendored/tre": ("laurikari/tre", None),
|
|
"vendored/xxhash": ("Cyan4973/xxHash", None),
|
|
"vendored/yyjson": ("ibireme/yyjson", None),
|
|
"internal/cbm/vendored/lz4": ("lz4/lz4", "lib/LICENSE"),
|
|
"internal/cbm/vendored/zstd": ("facebook/zstd", None),
|
|
"internal/cbm/vendored/simplecpp": ("danmar/simplecpp", None),
|
|
"internal/cbm/vendored/verstable": ("JacksonAllan/Verstable", None),
|
|
"internal/cbm/vendored/wyhash": ("wangyi-fudan/wyhash", None),
|
|
"internal/cbm/vendored/ts_runtime": ("tree-sitter/tree-sitter", None),
|
|
"internal/cbm/vendored/common": ("tree-sitter/tree-sitter-html", None),
|
|
"internal/cbm/vendored/common/tree_sitter": ("tree-sitter/tree-sitter", None),
|
|
}
|
|
CANDIDATE_NAMES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING",
|
|
"COPYING.txt", "LICENSE-MIT", "UNLICENSE", "LICENCE",
|
|
"license", "License.txt", "NOTICE"]
|
|
|
|
|
|
def gh_api(path):
|
|
r = subprocess.run(["gh", "api", path], capture_output=True, text=True)
|
|
if r.returncode != 0:
|
|
return None
|
|
return r.stdout
|
|
|
|
|
|
def upstream_default_license(repo):
|
|
out = gh_api(f"repos/{repo}/license")
|
|
if not out:
|
|
return None
|
|
try:
|
|
d = json.loads(out)
|
|
return base64.b64decode(d.get("content", "")).decode("utf-8", "replace")
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def upstream_file(repo, path, ref=None):
|
|
url = f"repos/{repo}/contents/{path}"
|
|
if ref:
|
|
url += f"?ref={ref}"
|
|
out = gh_api(url)
|
|
if not out:
|
|
return None
|
|
try:
|
|
d = json.loads(out)
|
|
if isinstance(d, dict) and d.get("content"):
|
|
return base64.b64decode(d["content"]).decode("utf-8", "replace")
|
|
except Exception:
|
|
pass
|
|
return None
|
|
|
|
|
|
def local_license(dirpath):
|
|
if not os.path.isdir(dirpath):
|
|
return None, None
|
|
for f in sorted(os.listdir(dirpath)):
|
|
if re.match(r"^(LICENSE|LICENCE|COPYING|UNLICENSE)", f, re.I):
|
|
p = os.path.join(dirpath, f)
|
|
with open(p, encoding="utf-8", errors="replace") as fh:
|
|
return f, fh.read()
|
|
return None, None
|
|
|
|
|
|
def parse_manifest():
|
|
"""grammar -> (repo, pinned_commit) from the verified-upstream table."""
|
|
out = {}
|
|
with open(os.path.join(GRAMMARS, "MANIFEST.md"), encoding="utf-8") as fh:
|
|
for line in fh:
|
|
m = re.match(
|
|
r"^\| (\w[\w.+-]*) \| \d+ \| ([\w.-]+/[\w.-]+) \| `([^`]+)` \|", line)
|
|
if m:
|
|
out[m.group(1)] = (m.group(2), m.group(3))
|
|
return out
|
|
|
|
|
|
def main():
|
|
with open(os.path.join(ROOT, "LICENSE"), encoding="utf-8") as fh:
|
|
root_license = fh.read()
|
|
|
|
manifest = parse_manifest()
|
|
results = {}
|
|
|
|
def check_upstream(key, dirpath, repo, pinned=None, exact_path=None):
|
|
fname, ours = local_license(dirpath)
|
|
if ours is None:
|
|
results[key] = ("NO-LOCAL-LICENSE", "")
|
|
return
|
|
# 1) exact upstream path (e.g. lz4 lib/LICENSE), at HEAD then pinned
|
|
if exact_path:
|
|
for ref in (None, pinned):
|
|
up = upstream_file(repo, exact_path, ref)
|
|
if up is not None and up == ours:
|
|
results[key] = ("IDENTICAL" if ref is None else "IDENTICAL@PINNED",
|
|
f"{repo}:{exact_path}")
|
|
return
|
|
# 2) default-branch detected license
|
|
up = upstream_default_license(repo)
|
|
if up is not None and up == ours:
|
|
results[key] = ("IDENTICAL", f"{repo} (default branch)")
|
|
return
|
|
# 3) pinned-commit candidates by filename
|
|
if pinned:
|
|
tried = [fname] + [c for c in CANDIDATE_NAMES if c != fname]
|
|
for cand in tried:
|
|
up2 = upstream_file(repo, cand, pinned)
|
|
if up2 is not None and up2 == ours:
|
|
results[key] = ("IDENTICAL@PINNED", f"{repo}:{cand}@{pinned}")
|
|
return
|
|
# 4) HEAD candidates by filename
|
|
for cand in CANDIDATE_NAMES:
|
|
up3 = upstream_file(repo, cand)
|
|
if up3 is not None and up3 == ours:
|
|
results[key] = ("IDENTICAL", f"{repo}:{cand} (default branch)")
|
|
return
|
|
results[key] = ("DIFFERS" if up is not None else "ERROR", f"{repo}")
|
|
|
|
# Libraries
|
|
for rel, (repo, exact) in LIBS.items():
|
|
check_upstream(rel, os.path.join(ROOT, rel), repo, exact_path=exact)
|
|
|
|
# Special: nomic = canonical Apache-2.0 text; sqlite3 = first-party notice
|
|
fname, ours = local_license(os.path.join(ROOT, "vendored/nomic"))
|
|
apache = subprocess.run(
|
|
["curl", "-fsSL", "https://www.apache.org/licenses/LICENSE-2.0.txt"],
|
|
capture_output=True, text=True).stdout
|
|
results["vendored/nomic"] = (
|
|
"IDENTICAL" if ours == apache else "DIFFERS",
|
|
"apache.org canonical LICENSE-2.0.txt")
|
|
results["vendored/sqlite3"] = ("FIRST-PARTY-NOTICE",
|
|
"own public-domain notice (sqlite has no upstream LICENSE)")
|
|
|
|
# Grammars
|
|
for g in sorted(os.listdir(GRAMMARS)):
|
|
d = os.path.join(GRAMMARS, g)
|
|
if not os.path.isdir(d):
|
|
continue
|
|
key = f"grammars/{g}"
|
|
if g in FIRST_PARTY:
|
|
fname, ours = local_license(d)
|
|
if ours == root_license:
|
|
results[key] = ("FIRST-PARTY-OK", "== project root LICENSE")
|
|
else:
|
|
results[key] = ("FIRST-PARTY-VAR", f"{fname}: differs from root LICENSE")
|
|
continue
|
|
if g in FORKS:
|
|
check_upstream(key, d, FORKS[g])
|
|
continue
|
|
if g in SPECIAL_NOTICE:
|
|
results[key] = ("MANUAL-VERIFIED", SPECIAL_NOTICE[g])
|
|
continue
|
|
if g in DISAGREEMENT:
|
|
check_upstream(key, d, DISAGREEMENT[g])
|
|
continue
|
|
if g in manifest:
|
|
repo, pinned = manifest[g]
|
|
check_upstream(key, d, repo, pinned=pinned)
|
|
continue
|
|
results[key] = ("NO-MANIFEST-ENTRY", "")
|
|
|
|
# Report
|
|
from collections import Counter
|
|
counts = Counter(v[0] for v in results.values())
|
|
print("=== verdict histogram ===")
|
|
for k, v in counts.most_common():
|
|
print(f" {k}: {v}")
|
|
print()
|
|
print("=== everything that is NOT byte-identical ===")
|
|
for key in sorted(results):
|
|
verdict, detail = results[key]
|
|
if verdict not in ("IDENTICAL", "IDENTICAL@PINNED", "FIRST-PARTY-OK"):
|
|
print(f" {key}: {verdict} [{detail}]")
|
|
json.dump({k: list(v) for k, v in results.items()},
|
|
open("/tmp/audit_licenses_results.json", "w"), indent=1)
|
|
print("\nfull results: /tmp/audit_licenses_results.json")
|
|
|
|
accepted = {"IDENTICAL", "IDENTICAL@PINNED", "FIRST-PARTY-OK",
|
|
"FIRST-PARTY-NOTICE", "MANUAL-VERIFIED"}
|
|
bad = {k: v for k, v in results.items() if v[0] not in accepted}
|
|
if bad:
|
|
print(f"\nPROVENANCE AUDIT FAILED: {len(bad)} unexplained verdict(s)")
|
|
sys.exit(1)
|
|
print("\nPROVENANCE AUDIT PASSED: every vendored license accounted for")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|