chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:12:00 +08:00
commit 3de48288cb
2986 changed files with 1131193 additions and 0 deletions
+163
View File
@@ -0,0 +1,163 @@
#!/usr/bin/env python3
"""Scan a PR's *added* lines for secret-exfiltration and obfuscated-exec shapes.
Part of the single contributor Security Scan (.github/workflows/security-scan.yml),
the companion to secret-scan.py: that one flags secrets a PR *commits*, this one
flags code a PR adds to *steal* the CI secrets it runs with (the test-gateway
token, GITHUB_TOKEN) -- an env-secret read piped to the network is the shape that
matters.
It reads diff TEXT only -- it never checks out or executes the PR's code -- so it
is safe on any event. It is defense-in-depth + a reviewer aid, NOT a guarantee:
an attacker can obfuscate past regexes, so maintainer review remains the primary
gate. Its job is to (a) hard-fail on high-confidence exfiltration shapes in ADDED
lines, and (b) surface changes to files that run during CI bootstrap so the
reviewer looks harder.
Findings are two tiers:
- BLOCKING -> non-zero exit: exfil shapes -- a secret-named credential source
AND a network sink added to the same file; a wholesale ``os.environ`` dump; a
decode-then-exec; or a raw TCP / reverse-shell sink.
- INFO -> ``::warning`` only: edits to CI-bootstrap-executed files (conftest.py,
setup.py, pyproject build hooks, anything under .github/, pytest plugins).
Env in: DIFF_FILE (path to a ``git diff base...head`` / ``gh pr diff`` unified diff).
Exit: non-zero if any BLOCKING finding; 0 otherwise.
"""
from __future__ import annotations
import os
import re
import sys
# Network / exfil sinks.
_NETWORK = re.compile(
r"requests\.(get|post|put|patch|request|Session)"
r"|urllib\.request|urlopen|httpx\.|aiohttp|http\.client"
r"|socket\.(socket|create_connection)|telnetlib|smtplib|ftplib"
r"|\bcurl\b|\bwget\b|\bnc\b|fetch\(|XMLHttpRequest|axios",
re.IGNORECASE,
)
# Secret-NAMED credential sources (deliberately narrow: generic os.environ /
# LLM_API_KEY use is normal in tests, so it is INFO-only, not blocking).
_SECRET = re.compile(
r"DATABRICKS_(CLIENT_ID|CLIENT_SECRET|TOKEN|BEARER)"
r"|FORK_E2E_APP_PRIVATE_KEY|PRIVATE_KEY|[A-Z0-9]+_SECRET\b"
# No bare ACCESS_TOKEN: case-insensitively it matches common `access_token`
# OAuth/JSON fields and would block legit PRs. The specific secret names
# above stay; generic-token exfil is left to the reviewer + LLM advisory.
r"|GITHUB_TOKEN|\bGH_TOKEN\b|\.databrickscfg",
re.IGNORECASE,
)
# Always-blocking single-line shapes (independent of co-occurrence).
_STANDALONE = re.compile(
r"/dev/tcp/" # bash reverse shell
# Wholesale environ dump only -- a bare `os.environ)` matched benign
# `helper(os.environ)` and is dropped to avoid false positives.
r"|(json\.dumps|dict|str|repr)\(\s*os\.environ" # dump the whole environ
r"|\beval\s*\(|\bexec\s*\(|__import__\s*\(" # dynamic exec
r"|pickle\.loads|marshal\.loads" # deserialization exec
r"|base64\.(b64decode|decodebytes)|codecs\.decode", # decode (paired below)
re.IGNORECASE,
)
_DECODE = re.compile(r"base64|b64decode|decodebytes|fromhex|codecs\.decode", re.IGNORECASE)
_EXEC = re.compile(
r"\beval\s*\(|\bexec\s*\(|__import__\s*\(|subprocess|os\.system|popen", re.IGNORECASE
)
# Files that execute during `uv sync` / pytest collection -- INFO, so the
# reviewer scrutinizes them even when no exfil pattern is present.
_HIGH_RISK = re.compile(
r"(^|/)conftest\.py$|(^|/)setup\.py$|(^|/)pyproject\.toml$"
r"|^\.github/|(^|/)sitecustomize\.py$|\.pth$"
r"|(^|/)_token_usage\.py$|(^|/)noxfile\.py$|(^|/)tox\.ini$|(^|/)Makefile$",
)
def _changed_files_and_added(diff: str) -> dict[str, list[str]]:
"""
Group a unified diff's ADDED lines by destination file.
:param diff: Full unified-diff text (e.g. from ``gh pr diff``).
:returns: Mapping of file path (e.g. ``"tests/conftest.py"``) to the list of
added line bodies (without the leading ``+``); diff headers excluded.
"""
by_file: dict[str, list[str]] = {}
current: str | None = None
for line in diff.splitlines():
if line.startswith("+++ b/"):
current = line[6:]
by_file.setdefault(current, [])
elif line.startswith(("+++ ", "diff --git")):
current = None
elif current is not None and line.startswith("+") and not line.startswith("+++"):
by_file[current].append(line[1:])
return by_file
def scan_diff(diff: str) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]:
"""
Classify a unified diff into blocking and info findings.
:param diff: Full unified-diff text.
:returns: ``(blocking, info)`` -- two lists of ``(path, message)`` tuples.
``blocking`` non-empty means the scan is not clean.
"""
by_file = _changed_files_and_added(diff)
blocking: list[tuple[str, str]] = []
info: list[tuple[str, str]] = []
for path, added in by_file.items():
body = "\n".join(added)
has_net = bool(_NETWORK.search(body))
has_secret = bool(_SECRET.search(body))
if has_net and has_secret:
blocking.append((path, "exfil shape: secret-named source + network sink in one file"))
for ln in added:
if _STANDALONE.search(ln) and not (
# a lone base64/decode call is INFO; only block decode+exec
_DECODE.search(ln) and not _EXEC.search(ln)
):
blocking.append((path, f"high-risk call: {ln.strip()[:80]}"))
break
if _DECODE.search(ln) and _EXEC.search(ln):
blocking.append((path, f"decode+exec: {ln.strip()[:80]}"))
break
if _HIGH_RISK.search(path):
info.append((path, "touches a file that runs during CI bootstrap; review closely"))
return blocking, info
def main() -> int:
"""
Scan the diff at ``$DIFF_FILE`` and report exfil / obfuscated-exec findings.
:returns: 1 if any blocking finding, else 0.
"""
diff_path = os.environ.get("DIFF_FILE")
if not diff_path or not os.path.isfile(diff_path):
print(f"::error::diff file {diff_path!r} missing")
return 1
with open(diff_path, encoding="utf-8", errors="replace") as fh:
diff = fh.read()
blocking, info = scan_diff(diff)
for path, msg in info:
print(f"::warning file={path}::{msg}")
for path, msg in blocking:
print(f"::error file={path}::{msg}")
if blocking:
print(f"::error::Exfil scan found {len(blocking)} blocking finding(s) in added lines.")
return 1
print(f"Exfil scan passed ({len(info)} CI-file note(s)).")
return 0
if __name__ == "__main__":
sys.exit(main())
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""Lint changed GitHub Actions workflows for the two highest-signal CI attacks.
Called by .github/workflows/security-gate.yml. Dependency-free (stdlib +
regex line scanning, no PyYAML) so it never needs a network install to run --
a security check should not depend on fetching anything.
Checks, per changed `.github/workflows/*.yml`:
1. pull_request_target + PR-head checkout (CRITICAL). The classic OSS
supply-chain RCE: a `pull_request_target` workflow runs from the base with
secrets, and if it also checks out / runs the PR head it executes
attacker code with secrets in scope. We flag any checkout that pulls a
PR-head ref (github.event.pull_request.head.*, github.head_ref,
refs/pull/...). A `# leak-scan-allow: pull_request_target` line (the
repo's existing convention for hand-audited exceptions) downgrades it to
a warning -- safe here because untrusted authors are independently blocked
from editing workflows by sensitive-paths.sh.
2. Unpinned action references (HIGH). `uses: owner/repo@v4` / `@main` lets the
action's owner change what runs under our token later. Require a 40-hex
commit SHA. Local (`./`) and `docker://...@sha256:` refs are exempt.
Env in: CHANGED_FILES (path to a file with one changed path per line).
Exit: non-zero if any CRITICAL/HIGH finding; 0 otherwise.
"""
from __future__ import annotations
import os
import re
import sys
SHA_RE = re.compile(r"^[0-9a-f]{40}$")
USES_RE = re.compile(r"""^\s*-?\s*uses:\s*['"]?([^'"\s#]+)['"]?""")
# PR-head refs that must never be checked out under pull_request_target.
HEAD_REF_RE = re.compile(
r"github\.event\.pull_request\.head\.(sha|ref)"
r"|github\.head_ref"
r"|refs/pull/",
)
def is_pinned(ref: str) -> bool:
if ref.startswith(("./", "../")):
return True # local action, ships with the repo
if ref.startswith("docker://"):
return "@sha256:" in ref # digest-pinned image
_, _, version = ref.partition("@")
return bool(SHA_RE.match(version))
def lint_file(path: str) -> tuple[list[str], list[str]]:
errors: list[str] = []
warnings: list[str] = []
try:
with open(path, encoding="utf-8") as fh:
text = fh.read()
except OSError as e:
warnings.append(f"::warning file={path}::could not read workflow ({e})")
return errors, warnings
lines = text.splitlines()
allow_prt = "leak-scan-allow: pull_request_target" in text
has_prt = re.search(r"^\s*pull_request_target\s*:", text, re.MULTILINE) is not None
for i, line in enumerate(lines, 1):
if line.lstrip().startswith("#"):
continue
# 1. PR-head checkout under pull_request_target.
if has_prt and HEAD_REF_RE.search(line):
msg = (
f"file={path},line={i}::pull_request_target workflow references a "
"PR-head ref -- this runs untrusted PR code with secrets. "
"Check out 'main' only, or read the PR via the API."
)
(warnings if allow_prt else errors).append(
("::warning " if allow_prt else "::error ") + msg
)
# 2. Unpinned action reference.
m = USES_RE.match(line)
if m:
ref = m.group(1)
if "@" in ref and not is_pinned(ref):
errors.append(
f"::error file={path},line={i}::action '{ref}' is not pinned to a "
"full commit SHA; a tag/branch ref can be moved to hostile code."
)
return errors, warnings
def main() -> int:
changed = os.environ.get("CHANGED_FILES")
if not changed or not os.path.isfile(changed):
print(f"::error::changed-files list {changed!r} missing")
return 1
with open(changed, encoding="utf-8") as fh:
paths = [p.strip() for p in fh if p.strip()]
targets = [
p
for p in paths
if p.startswith(".github/workflows/")
and p.endswith((".yml", ".yaml"))
and os.path.isfile(p)
]
if not targets:
print("No changed workflow files to lint.")
return 0
all_errors: list[str] = []
for path in targets:
errors, warnings = lint_file(path)
for w in warnings:
print(w)
for e in errors:
print(e)
all_errors.extend(errors)
if all_errors:
print(f"::error::Workflow misuse linter failed with {len(all_errors)} finding(s).")
return 1
print(f"Workflow misuse linter passed ({len(targets)} file(s) checked).")
return 0
if __name__ == "__main__":
sys.exit(main())
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
"""Scan a PR's *added* lines for committed secrets.
Called by .github/workflows/security-gate.yml. Dependency-free (stdlib only)
so it runs without a network install. Operates on a unified diff and inspects
only added (`+`) lines, so it flags secrets the PR introduces, not pre-existing
ones -- and reports them at the right file/line for inline annotations.
Detection is two-pronged:
* High-confidence provider token shapes (AWS, GitHub, Slack, Google, private
keys) -- low false-positive, reported as errors.
* Generic high-entropy assignments to secret-looking names
(token/secret/password/api_key=...) -- reported as errors when the value is
long and high-entropy.
This is intentionally a curated, hermetic baseline, not a replacement for
gitleaks/trufflehog; those can be layered in later once an org license / pinned
action SHA is settled (see plan).
Env in: DIFF_FILE (path to a `git diff base...head` unified diff).
Exit: non-zero if any secret is found; 0 otherwise.
"""
from __future__ import annotations
import math
import os
import re
import sys
HIGH_CONFIDENCE = [
("AWS access key id", re.compile(r"\b(AKIA|ASIA)[0-9A-Z]{16}\b")),
("GitHub token", re.compile(r"\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b")),
("GitHub fine-grained PAT", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{60,}\b")),
("Slack token", re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b")),
("Google API key", re.compile(r"\bAIza[0-9A-Za-z_\-]{35}\b")),
(
"private key block",
re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----"),
),
("Stripe secret key", re.compile(r"\b(sk|rk)_live_[0-9A-Za-z]{24,}\b")),
]
# name = "value" / name: value / name=value for secret-ish names.
ASSIGN_RE = re.compile(
r"""(?ix)
\b(?P<name>[a-z0-9_\-\.]*(?:secret|token|passwd|password|api[_\-]?key|access[_\-]?key|private[_\-]?key)[a-z0-9_\-\.]*)
\s*[:=]\s*
['"]?(?P<value>[A-Za-z0-9+/_\-\.=]{20,})['"]?
"""
)
# Values that look like references/placeholders, not real secrets.
PLACEHOLDER_RE = re.compile(
r"(?i)\$\{|\$\(|secrets\.|env\.|vars\.|os\.environ|getenv|process\.env"
r"|example|placeholder|changeme|your[_\-]?|xxx|<.*>|\*{4,}|redacted|dummy|fake|todo"
)
def shannon_entropy(s: str) -> float:
if not s:
return 0.0
counts = {c: s.count(c) for c in set(s)}
n = len(s)
return -sum((c / n) * math.log2(c / n) for c in counts.values())
def scan_value(value: str) -> bool:
"""Generic heuristic: long, high-entropy, not an obvious placeholder."""
if PLACEHOLDER_RE.search(value):
return False
if len(value) < 20:
return False
return shannon_entropy(value) >= 4.0
def main() -> int:
diff_path = os.environ.get("DIFF_FILE")
if not diff_path or not os.path.isfile(diff_path):
print(f"::error::diff file {diff_path!r} missing")
return 1
findings: list[str] = []
cur_file = "?"
new_lineno = 0
with open(diff_path, encoding="utf-8", errors="replace") as fh:
for raw in fh:
line = raw.rstrip("\n")
if line.startswith("+++ "):
cur_file = line[6:] if line.startswith("+++ b/") else line[4:]
continue
if line.startswith("@@"):
m = re.search(r"\+(\d+)", line)
new_lineno = int(m.group(1)) if m else 0
continue
if line.startswith("+") and not line.startswith("+++"):
added = line[1:]
for label, rx in HIGH_CONFIDENCE:
if rx.search(added):
findings.append(
f"::error file={cur_file},line={new_lineno}::"
f"possible committed secret ({label})."
)
break
else:
m = ASSIGN_RE.search(added)
if m and scan_value(m.group("value")):
findings.append(
f"::error file={cur_file},line={new_lineno}::"
f"possible hardcoded secret assigned to '{m.group('name')}' "
"(long, high-entropy value)."
)
new_lineno += 1
elif not line.startswith("-"):
# context line advances the new-file counter too
new_lineno += 1
for f in findings:
print(f)
if findings:
print(f"::error::Secret scan found {len(findings)} candidate secret(s) in added lines.")
return 1
print("Secret scan passed (no secrets in added lines).")
return 0
if __name__ == "__main__":
sys.exit(main())
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
# Flags PR changes to security-sensitive paths. Called by
# .github/workflows/security-gate.yml after the trust gate opens.
#
# Two tiers:
# FAIL -- paths that let a PR escalate privilege or rewrite the trust model:
# CI workflows, the maintainer list, code owners. An untrusted
# author has no business editing these; a real need is unblocked by
# a maintainer reviewing and merging the change anyway.
# WARN -- build/test hooks that execute code at install or collection time
# (setup.py, pyproject build backends, conftest.py) and the lockfile.
# Not auto-failed (legit PRs touch them), but surfaced as annotations
# so a reviewer looks closely. semgrep + the secret scan still run on
# their contents.
#
# Env in: CHANGED_FILES (path to a file with one changed path per line).
# Exit: non-zero if any FAIL-tier path changed; 0 otherwise.
set -euo pipefail
CHANGED="${CHANGED_FILES:?CHANGED_FILES not set}"
[[ -f "$CHANGED" ]] || { echo "::error::changed-files list $CHANGED missing"; exit 1; }
fail=0
while IFS= read -r path; do
[[ -z "$path" ]] && continue
case "$path" in
.github/workflows/*)
echo "::error file=$path::Untrusted PR edits a CI workflow. Workflow changes can exfiltrate secrets or weaken gates; a maintainer must review."
fail=1
;;
.github/MAINTAINER)
echo "::error file=$path::Untrusted PR edits .github/MAINTAINER (the maintainer allowlist). Self-granting maintainership is blocked."
fail=1
;;
.github/CODEOWNERS | CODEOWNERS | docs/CODEOWNERS)
echo "::error file=$path::Untrusted PR edits CODEOWNERS. Review-routing changes must be made by a maintainer."
fail=1
;;
.github/scripts/*)
echo "::error file=$path::Untrusted PR edits a CI helper script under .github/scripts. These run in privileged workflows; a maintainer must review."
fail=1
;;
setup.py | */setup.py | pyproject.toml | */pyproject.toml | conftest.py | */conftest.py)
echo "::warning file=$path::PR edits a build/test hook that runs code at install or collection time. Review for code execution side effects."
;;
uv.lock | */uv.lock | package-lock.json | */package-lock.json | yarn.lock | */yarn.lock)
echo "::warning file=$path::PR edits a dependency lockfile. Review for dependency-confusion / typosquat / repointed sources."
;;
esac
done < "$CHANGED"
if [[ "$fail" -ne 0 ]]; then
echo "::error::Sensitive-path guard failed: this PR modifies privileged repo configuration."
exit 1
fi
echo "Sensitive-path guard passed."
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env bash
# Decides whether a PR's diff should be put through the Security Scan.
# Called by .github/workflows/security-gate.yml.
#
# We scan UNTRUSTED authors and skip trusted ones. "Trusted" is GitHub's
# native author_association: OWNER / MEMBER / COLLABORATOR -- people with a
# direct relationship to the repo/org -- OR an author in the MAINTAINERS list.
# The list covers maintainers whose org membership is PRIVATE: GitHub only
# reports MEMBER in author_association when membership is public, so a private
# maintainer shows up as CONTRIBUTOR and would otherwise be scanned. Everyone
# else is scanned, INCLUDING returning CONTRIBUTORs (a merged PR in the past
# does not vouch for the contents of this one) and first-timers
# (FIRST_TIME_CONTRIBUTOR / NONE).
#
# This gate decides whether to inspect a PR for attacks and errs toward scanning
# more (it scans returning CONTRIBUTORs, not just first-timers).
#
# author_association is computed by GitHub from the actor's relationship to the
# repo at event time; it is not attacker-settable from PR contents.
#
# Maintainer escape hatch: an untrusted PR can be waived by the
# `skip-security-scan` label alone. Applying a label requires GitHub Triage
# permission (or higher), which a fork author never has, so the label IS the
# maintainer gate and no separate approval is required.
#
# ACCEPTED RISK (repo policy, not GitHub-enforced): GitHub allows the Triage role
# to be granted independently of Write, so in principle a triage-only collaborator
# could self-waive. We accept this because this repo grants Triage only to
# write/admin collaborators -- everyone who can apply the label can already push
# code, so the waiver grants no privilege they don't already have. This invariant
# lives in repo settings, not in code; if Triage is ever granted without Write,
# revisit (e.g. re-add a maintainer-list check). See the PR for the full rationale.
#
# The label is read from the API (trusted), and this script always runs from
# `main`, so a PR cannot edit the decision. The waiver is only evaluated when the
# lookup vars (GH_TOKEN/REPO/PR) are passed (the scan does; the per-workflow
# pollers do not -- they just mirror the scan's result).
#
# Env in: EVENT_NAME (github.event_name)
# AUTHOR_ASSOCIATION (github.event.pull_request.author_association)
# MAINTAINERS (space-separated, from merge-ready/load-maintainers.sh;
# optional -- used only to trust private-membership
# maintainer AUTHORS, not for the label waiver)
# GH_TOKEN, REPO, PR (for the label lookup + author check)
# Out: `scan=true|false` and `reason=<text>` on $GITHUB_OUTPUT.
set -euo pipefail
SKIP_LABEL="skip-security-scan"
emit() {
echo "scan=$1" >> "$GITHUB_OUTPUT"
echo "reason=$2" >> "$GITHUB_OUTPUT"
echo "scan=$1 ($2)"
}
# 0 = the skip label is present; 1 otherwise. Label-only: applying the label
# already requires Triage permission (or higher), so its mere presence is the
# maintainer gate (see the accepted-risk note in the header). Fails closed on any
# gap (missing token, etc).
has_skip_label() {
[[ -n "${GH_TOKEN:-}" && -n "${REPO:-}" && -n "${PR:-}" ]] || return 1
local has_label
has_label=$(gh api "repos/$REPO/pulls/$PR" \
--jq "[.labels[].name] | index(\"$SKIP_LABEL\") != null" 2>/dev/null || echo "false")
[[ "$has_label" == "true" ]]
}
# Only PRs carry untrusted contributor code through the gate. Every other
# trigger -- push to main, schedule, dispatch -- is a trusted context, so
# proceed without scanning. pull_request_review is still
# accepted (it carries the same pull_request + author_association fields, so the
# gate evaluates identically) in case a workflow_call caller is wired to it, but
# no workflow triggers a scan on review any more: the skip-security-scan waiver
# is label-only, so the label event alone re-runs the scan and flips the check.
case "${EVENT_NAME:-}" in
pull_request | pull_request_target | pull_request_review) ;;
*)
emit false "non-PR event (${EVENT_NAME:-unknown}); trusted context"
exit 0
;;
esac
# Author is a known maintainer? `author_association` only reports MEMBER when
# the org membership is PUBLIC, so a maintainer with private membership shows up
# as CONTRIBUTOR in the event payload and would otherwise be scanned. The
# MAINTAINERS list (from load-maintainers.sh) is authoritative and trusted, so
# trust the author directly when they appear in it. Only evaluated when
# MAINTAINERS is passed (the scan does; the per-workflow pollers do not).
author_is_maintainer() {
[[ -n "${MAINTAINERS:-}" && -n "${MAINTAINERS// /}" ]] || return 1
[[ -n "${GH_TOKEN:-}" && -n "${REPO:-}" && -n "${PR:-}" ]] || return 1
local maint_lc author_lc
maint_lc=$(echo "$MAINTAINERS" | tr '[:upper:]' '[:lower:]')
author_lc=$(gh pr view "$PR" --repo "$REPO" --json author --jq '.author.login' 2>/dev/null \
| tr '[:upper:]' '[:lower:]')
[[ -n "$author_lc" ]] || return 1
for m in $maint_lc; do
[[ "$m" == "$author_lc" ]] && return 0
done
return 1
}
case "${AUTHOR_ASSOCIATION:-}" in
OWNER | MEMBER | COLLABORATOR)
emit false "trusted author (author_association=$AUTHOR_ASSOCIATION)"
;;
*)
if author_is_maintainer; then
emit false "trusted author (maintainer; author_association=${AUTHOR_ASSOCIATION:-unknown})"
elif has_skip_label; then
emit false "'$SKIP_LABEL' waiver (label requires a Triage+ collaborator to apply)"
else
emit true "untrusted author (author_association=${AUTHOR_ASSOCIATION:-unknown})"
fi
;;
esac