chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+226
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Byte-identity audit: compare every vendored license file against upstream.
|
||||
|
||||
Verdicts:
|
||||
IDENTICAL byte-equal to the upstream default-branch license
|
||||
IDENTICAL@PINNED byte-equal to the license at the manifest's pinned commit
|
||||
FIRST-PARTY-OK byte-equal to the project root LICENSE
|
||||
FIRST-PARTY-VAR first-party but text differs from root LICENSE (inspect)
|
||||
DIFFERS no byte-equal upstream candidate found (inspect)
|
||||
ERROR upstream fetch failed
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
GRAMMARS = os.path.join(ROOT, "internal/cbm/vendored/grammars")
|
||||
|
||||
FIRST_PARTY = {"cobol", "form", "janet", "magma", "protobuf", "wolfram"}
|
||||
FORKS = { # self-maintained forks: vendored LICENSE must match the original upstream
|
||||
"cfml": "cfmleditor/tree-sitter-cfml",
|
||||
"cfscript": "cfmleditor/tree-sitter-cfml",
|
||||
"dotenv": "pnx/tree-sitter-dotenv",
|
||||
"qml": "yuja/tree-sitter-qmljs",
|
||||
}
|
||||
SPECIAL_NOTICE = {
|
||||
"assembly": "RETAINED-MIT (upstream RubixDev/tree-sitter-assembly deleted from GitHub)",
|
||||
"pine": "PROVENANCE-NOTICE (upstream kvarenzn/tree-sitter-pine declares ISC, ships no license file)",
|
||||
}
|
||||
DISAGREEMENT = {
|
||||
"jinja2": "dbt-labs/tree-sitter-jinja2",
|
||||
"just": "casey/tree-sitter-just",
|
||||
"move": "tzakian/tree-sitter-move",
|
||||
"sshconfig": "ObserverOfTime/tree-sitter-ssh-config",
|
||||
"zsh": "georgeharker/tree-sitter-zsh",
|
||||
}
|
||||
LIBS = {
|
||||
"vendored/mimalloc": ("microsoft/mimalloc", None),
|
||||
"vendored/tre": ("laurikari/tre", None),
|
||||
"vendored/xxhash": ("Cyan4973/xxHash", None),
|
||||
"vendored/yyjson": ("ibireme/yyjson", None),
|
||||
"internal/cbm/vendored/lz4": ("lz4/lz4", "lib/LICENSE"),
|
||||
"internal/cbm/vendored/zstd": ("facebook/zstd", None),
|
||||
"internal/cbm/vendored/simplecpp": ("danmar/simplecpp", None),
|
||||
"internal/cbm/vendored/verstable": ("JacksonAllan/Verstable", None),
|
||||
"internal/cbm/vendored/wyhash": ("wangyi-fudan/wyhash", None),
|
||||
"internal/cbm/vendored/ts_runtime": ("tree-sitter/tree-sitter", None),
|
||||
"internal/cbm/vendored/common": ("tree-sitter/tree-sitter-html", None),
|
||||
"internal/cbm/vendored/common/tree_sitter": ("tree-sitter/tree-sitter", None),
|
||||
}
|
||||
CANDIDATE_NAMES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING",
|
||||
"COPYING.txt", "LICENSE-MIT", "UNLICENSE", "LICENCE",
|
||||
"license", "License.txt", "NOTICE"]
|
||||
|
||||
|
||||
def gh_api(path):
|
||||
r = subprocess.run(["gh", "api", path], capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
return r.stdout
|
||||
|
||||
|
||||
def upstream_default_license(repo):
|
||||
out = gh_api(f"repos/{repo}/license")
|
||||
if not out:
|
||||
return None
|
||||
try:
|
||||
d = json.loads(out)
|
||||
return base64.b64decode(d.get("content", "")).decode("utf-8", "replace")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def upstream_file(repo, path, ref=None):
|
||||
url = f"repos/{repo}/contents/{path}"
|
||||
if ref:
|
||||
url += f"?ref={ref}"
|
||||
out = gh_api(url)
|
||||
if not out:
|
||||
return None
|
||||
try:
|
||||
d = json.loads(out)
|
||||
if isinstance(d, dict) and d.get("content"):
|
||||
return base64.b64decode(d["content"]).decode("utf-8", "replace")
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def local_license(dirpath):
|
||||
if not os.path.isdir(dirpath):
|
||||
return None, None
|
||||
for f in sorted(os.listdir(dirpath)):
|
||||
if re.match(r"^(LICENSE|LICENCE|COPYING|UNLICENSE)", f, re.I):
|
||||
p = os.path.join(dirpath, f)
|
||||
with open(p, encoding="utf-8", errors="replace") as fh:
|
||||
return f, fh.read()
|
||||
return None, None
|
||||
|
||||
|
||||
def parse_manifest():
|
||||
"""grammar -> (repo, pinned_commit) from the verified-upstream table."""
|
||||
out = {}
|
||||
with open(os.path.join(GRAMMARS, "MANIFEST.md"), encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
m = re.match(
|
||||
r"^\| (\w[\w.+-]*) \| \d+ \| ([\w.-]+/[\w.-]+) \| `([^`]+)` \|", line)
|
||||
if m:
|
||||
out[m.group(1)] = (m.group(2), m.group(3))
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
with open(os.path.join(ROOT, "LICENSE"), encoding="utf-8") as fh:
|
||||
root_license = fh.read()
|
||||
|
||||
manifest = parse_manifest()
|
||||
results = {}
|
||||
|
||||
def check_upstream(key, dirpath, repo, pinned=None, exact_path=None):
|
||||
fname, ours = local_license(dirpath)
|
||||
if ours is None:
|
||||
results[key] = ("NO-LOCAL-LICENSE", "")
|
||||
return
|
||||
# 1) exact upstream path (e.g. lz4 lib/LICENSE), at HEAD then pinned
|
||||
if exact_path:
|
||||
for ref in (None, pinned):
|
||||
up = upstream_file(repo, exact_path, ref)
|
||||
if up is not None and up == ours:
|
||||
results[key] = ("IDENTICAL" if ref is None else "IDENTICAL@PINNED",
|
||||
f"{repo}:{exact_path}")
|
||||
return
|
||||
# 2) default-branch detected license
|
||||
up = upstream_default_license(repo)
|
||||
if up is not None and up == ours:
|
||||
results[key] = ("IDENTICAL", f"{repo} (default branch)")
|
||||
return
|
||||
# 3) pinned-commit candidates by filename
|
||||
if pinned:
|
||||
tried = [fname] + [c for c in CANDIDATE_NAMES if c != fname]
|
||||
for cand in tried:
|
||||
up2 = upstream_file(repo, cand, pinned)
|
||||
if up2 is not None and up2 == ours:
|
||||
results[key] = ("IDENTICAL@PINNED", f"{repo}:{cand}@{pinned}")
|
||||
return
|
||||
# 4) HEAD candidates by filename
|
||||
for cand in CANDIDATE_NAMES:
|
||||
up3 = upstream_file(repo, cand)
|
||||
if up3 is not None and up3 == ours:
|
||||
results[key] = ("IDENTICAL", f"{repo}:{cand} (default branch)")
|
||||
return
|
||||
results[key] = ("DIFFERS" if up is not None else "ERROR", f"{repo}")
|
||||
|
||||
# Libraries
|
||||
for rel, (repo, exact) in LIBS.items():
|
||||
check_upstream(rel, os.path.join(ROOT, rel), repo, exact_path=exact)
|
||||
|
||||
# Special: nomic = canonical Apache-2.0 text; sqlite3 = first-party notice
|
||||
fname, ours = local_license(os.path.join(ROOT, "vendored/nomic"))
|
||||
apache = subprocess.run(
|
||||
["curl", "-fsSL", "https://www.apache.org/licenses/LICENSE-2.0.txt"],
|
||||
capture_output=True, text=True).stdout
|
||||
results["vendored/nomic"] = (
|
||||
"IDENTICAL" if ours == apache else "DIFFERS",
|
||||
"apache.org canonical LICENSE-2.0.txt")
|
||||
results["vendored/sqlite3"] = ("FIRST-PARTY-NOTICE",
|
||||
"own public-domain notice (sqlite has no upstream LICENSE)")
|
||||
|
||||
# Grammars
|
||||
for g in sorted(os.listdir(GRAMMARS)):
|
||||
d = os.path.join(GRAMMARS, g)
|
||||
if not os.path.isdir(d):
|
||||
continue
|
||||
key = f"grammars/{g}"
|
||||
if g in FIRST_PARTY:
|
||||
fname, ours = local_license(d)
|
||||
if ours == root_license:
|
||||
results[key] = ("FIRST-PARTY-OK", "== project root LICENSE")
|
||||
else:
|
||||
results[key] = ("FIRST-PARTY-VAR", f"{fname}: differs from root LICENSE")
|
||||
continue
|
||||
if g in FORKS:
|
||||
check_upstream(key, d, FORKS[g])
|
||||
continue
|
||||
if g in SPECIAL_NOTICE:
|
||||
results[key] = ("MANUAL-VERIFIED", SPECIAL_NOTICE[g])
|
||||
continue
|
||||
if g in DISAGREEMENT:
|
||||
check_upstream(key, d, DISAGREEMENT[g])
|
||||
continue
|
||||
if g in manifest:
|
||||
repo, pinned = manifest[g]
|
||||
check_upstream(key, d, repo, pinned=pinned)
|
||||
continue
|
||||
results[key] = ("NO-MANIFEST-ENTRY", "")
|
||||
|
||||
# Report
|
||||
from collections import Counter
|
||||
counts = Counter(v[0] for v in results.values())
|
||||
print("=== verdict histogram ===")
|
||||
for k, v in counts.most_common():
|
||||
print(f" {k}: {v}")
|
||||
print()
|
||||
print("=== everything that is NOT byte-identical ===")
|
||||
for key in sorted(results):
|
||||
verdict, detail = results[key]
|
||||
if verdict not in ("IDENTICAL", "IDENTICAL@PINNED", "FIRST-PARTY-OK"):
|
||||
print(f" {key}: {verdict} [{detail}]")
|
||||
json.dump({k: list(v) for k, v in results.items()},
|
||||
open("/tmp/audit_licenses_results.json", "w"), indent=1)
|
||||
print("\nfull results: /tmp/audit_licenses_results.json")
|
||||
|
||||
accepted = {"IDENTICAL", "IDENTICAL@PINNED", "FIRST-PARTY-OK",
|
||||
"FIRST-PARTY-NOTICE", "MANUAL-VERIFIED"}
|
||||
bad = {k: v for k, v in results.items() if v[0] not in accepted}
|
||||
if bad:
|
||||
print(f"\nPROVENANCE AUDIT FAILED: {len(bad)} unexplained verdict(s)")
|
||||
sys.exit(1)
|
||||
print("\nPROVENANCE AUDIT PASSED: every vendored license accounted for")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user