shiahonb777--web-to-app/.github/scripts/ci/validate_modules.py

#!/usr/bin/env python3
"""
Validate the WebToApp Module Market catalog at `modules/`.

This is the source of truth for what `modules/README.md` claims gets
checked at CI time. Every rule here corresponds to a real failure mode
that would either reject a module at install time or quietly waste space
on disk.

The validator is intentionally written with only Python's standard
library — no pip install step, no extra container — so the CI job stays
fast and deterministic.

Run locally:

    python3 .github/scripts/ci/validate_modules.py

Exit codes:
    0 = catalog is valid
    1 = at least one error was found (CI fails)
"""

from __future__ import annotations

import json
import os
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable


# ───────────────────────── enum values (must mirror the Kotlin source) ─

# `ModuleCategory` enum in `core/extension/ExtensionModule.kt`.
ALLOWED_CATEGORIES: set[str] = {
    "CONTENT_FILTER", "CONTENT_ENHANCE", "STYLE_MODIFIER", "THEME",
    "FUNCTION_ENHANCE", "AUTOMATION", "NAVIGATION", "DATA_EXTRACT",
    "DATA_SAVE", "INTERACTION", "ACCESSIBILITY", "MEDIA", "VIDEO",
    "IMAGE", "AUDIO", "SECURITY", "ANTI_TRACKING", "SOCIAL", "SHOPPING",
    "READING", "TRANSLATE", "DEVELOPER", "OTHER",
}

# `ModuleRunTime` enum.
ALLOWED_RUN_AT: set[str] = {
    "DOCUMENT_START", "DOCUMENT_END", "DOCUMENT_IDLE",
    "CONTEXT_MENU", "BEFORE_UNLOAD",
}

# `ModulePermission` enum.
ALLOWED_PERMISSIONS: set[str] = {
    "DOM_ACCESS", "DOM_OBSERVE", "CSS_INJECT", "STORAGE", "COOKIE",
    "INDEXED_DB", "CACHE", "NETWORK", "WEBSOCKET", "FETCH_INTERCEPT",
    "CLIPBOARD", "NOTIFICATION", "ALERT", "KEYBOARD", "MOUSE", "TOUCH",
    "LOCATION", "CAMERA", "MICROPHONE", "DEVICE_INFO", "MEDIA",
    "FULLSCREEN", "PICTURE_IN_PICTURE", "SCREEN_CAPTURE", "DOWNLOAD",
    "FILE_ACCESS", "EVAL", "IFRAME", "WINDOW_OPEN", "HISTORY",
    "NAVIGATION",
}

# `ConfigItemType` enum.
ALLOWED_CONFIG_TYPES: set[str] = {
    "TEXT", "TEXTAREA", "NUMBER", "BOOLEAN", "SELECT", "MULTI_SELECT",
    "RADIO", "CHECKBOX", "COLOR", "URL", "EMAIL", "PASSWORD", "REGEX",
    "CSS_SELECTOR", "JAVASCRIPT", "JSON", "RANGE", "DATE", "TIME",
    "DATETIME", "FILE", "IMAGE",
}

ALLOWED_SOURCE_TYPES: set[str] = {"CUSTOM", "CHROME_EXTENSION"}
STORE_ID_RE = re.compile(r"^[a-p]{32}$")


# ───────────────────────── regex helpers ───────────────────────────────

KEBAB_CASE_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
SEMVER_RE = re.compile(r"^\d+(?:\.\d+){0,3}(?:[-+][\w.-]+)?$")

# Files we let people drop in besides the three the runtime actually
# downloads (module.json, main.js, style.css).
ALLOWED_EXTRA_FILES: set[str] = {
    "README.md",       # nice for module pages on GitHub
    "CHANGELOG.md",
    "LICENSE",
    "LICENSE.md",
    ".gitkeep",
}

# Image files contributors may drop next to `main.js` for use as the
# module icon. The CI generator does not download these — they are served
# via the same GitHub raw / jsDelivr fallback the runtime already uses.
ALLOWED_ICON_FILES: set[str] = {
    "icon.png", "icon.svg", "icon.webp", "icon.jpg", "icon.jpeg",
}

# Maximum size for an inlined icon. Larger icons should be hosted off-repo
# and referenced via an absolute `iconUrl`. 256 KB is a forgiving limit:
# a 256x256 PNG with reasonable compression sits comfortably below it.
MAX_ICON_BYTES = 256 * 1024


@dataclass
class Report:
    """Accumulates diagnostics and pretty-prints them at the end."""

    errors: list[str] = field(default_factory=list)
    warnings: list[str] = field(default_factory=list)

    def error(self, where: str, message: str) -> None:
        self.errors.append(f"  ❌ {where}: {message}")

    def warning(self, where: str, message: str) -> None:
        self.warnings.append(f"  ⚠️  {where}: {message}")

    def ok(self) -> bool:
        return not self.errors

    def render(self) -> str:
        lines: list[str] = []
        if self.errors:
            lines.append(f"\n{len(self.errors)} error(s):")
            lines.extend(self.errors)
        if self.warnings:
            lines.append(f"\n{len(self.warnings)} warning(s):")
            lines.extend(self.warnings)
        if not lines:
            lines.append("\nAll module checks passed.")
        return "\n".join(lines)


# ───────────────────────── primitive checks ────────────────────────────

def _is_str(value: Any) -> bool:
    return isinstance(value, str)


def _expect_str(report: Report, where: str, value: Any, field: str) -> str | None:
    if value is None:
        return None
    if not _is_str(value):
        report.error(where, f"`{field}` must be a string, got {type(value).__name__}")
        return None
    return value


def _expect_list_of_str(report: Report, where: str, value: Any, field: str) -> list[str]:
    if value is None:
        return []
    if not isinstance(value, list) or any(not _is_str(v) for v in value):
        report.error(where, f"`{field}` must be a list of strings")
        return []
    return value


def _validate_url_matches(
    report: Report, where: str, url_matches: Any
) -> None:
    if url_matches is None:
        return
    if not isinstance(url_matches, list):
        report.error(where, "`urlMatches` must be a list")
        return
    for i, rule in enumerate(url_matches):
        rule_loc = f"{where}::urlMatches[{i}]"
        if not isinstance(rule, dict):
            report.error(rule_loc, "must be an object with at least `pattern`")
            continue
        pattern = rule.get("pattern")
        if not _is_str(pattern) or not pattern:
            report.error(rule_loc, "`pattern` is required and must be a non-empty string")
        for flag in ("isRegex", "exclude"):
            if flag in rule and not isinstance(rule[flag], bool):
                report.error(rule_loc, f"`{flag}` must be a boolean")


def _validate_author(report: Report, where: str, author: Any) -> None:
    if author is None:
        return
    if not isinstance(author, dict):
        report.error(where, "`author` must be an object")
        return
    name = author.get("name")
    if not _is_str(name) or not name.strip():
        report.error(where, "`author.name` is required")
    for opt_key in ("email", "url", "qq"):
        if opt_key in author and author[opt_key] is not None and not _is_str(author[opt_key]):
            report.error(where, f"`author.{opt_key}` must be a string")


# ───────────────────────── manifest checks ─────────────────────────────

def _validate_module_json(
    report: Report, folder: Path, manifest: dict[str, Any]
) -> None:
    """Validate a per-module `module.json`."""

    where = f"modules/{folder.name}/module.json"

    # Required fields.
    if not _is_str(manifest.get("id")) or not manifest["id"].strip():
        report.error(where, "`id` is required and must be a non-empty string")
    if not _is_str(manifest.get("name")) or not manifest["name"].strip():
        report.error(where, "`name` is required and must be a non-empty string")

    # Enums (with a friendly hint when wrong).
    category = manifest.get("category", "OTHER")
    if not _is_str(category) or category not in ALLOWED_CATEGORIES:
        report.error(
            where,
            f"`category` must be one of {sorted(ALLOWED_CATEGORIES)}, got {category!r}",
        )

    run_at = manifest.get("runAt", "DOCUMENT_END")
    if not _is_str(run_at) or run_at not in ALLOWED_RUN_AT:
        report.error(
            where,
            f"`runAt` must be one of {sorted(ALLOWED_RUN_AT)}, got {run_at!r}",
        )

    # Version (object form).
    version = manifest.get("version")
    if version is None:
        report.error(where, "`version` is required (object with `code`, `name`, `changelog`)")
    elif not isinstance(version, dict):
        report.error(where, "`version` must be an object — see modules/README.md schema")
    else:
        v_code = version.get("code")
        if not isinstance(v_code, int) or v_code < 1:
            report.error(where, "`version.code` must be a positive integer")
        v_name = version.get("name")
        if not _is_str(v_name) or not SEMVER_RE.match(v_name or ""):
            report.error(where, f"`version.name` must be semver-shaped, got {v_name!r}")
        if "changelog" in version and version["changelog"] is not None and not _is_str(version["changelog"]):
            report.error(where, "`version.changelog` must be a string")

    # Permissions.
    for perm in _expect_list_of_str(report, where, manifest.get("permissions"), "permissions"):
        if perm not in ALLOWED_PERMISSIONS:
            report.error(where, f"unknown permission {perm!r}")

    # urlMatches and author.
    _validate_url_matches(report, where, manifest.get("urlMatches"))
    _validate_author(report, where, manifest.get("author"))

    # configItems.
    config_items = manifest.get("configItems", [])
    if config_items is None:
        config_items = []
    if not isinstance(config_items, list):
        report.error(where, "`configItems` must be a list")
        return
    seen_keys: set[str] = set()
    for i, item in enumerate(config_items):
        item_loc = f"{where}::configItems[{i}]"
        if not isinstance(item, dict):
            report.error(item_loc, "must be an object")
            continue
        key = item.get("key")
        if not _is_str(key) or not key.strip():
            report.error(item_loc, "`key` is required")
        elif key in seen_keys:
            report.error(item_loc, f"duplicate key {key!r}")
        else:
            seen_keys.add(key)

        if not _is_str(item.get("name")) or not item["name"].strip():
            report.error(item_loc, "`name` is required")

        type_ = item.get("type", "TEXT")
        if not _is_str(type_) or type_ not in ALLOWED_CONFIG_TYPES:
            report.error(item_loc, f"`type` must be one of {sorted(ALLOWED_CONFIG_TYPES)}")

        if "required" in item and not isinstance(item["required"], bool):
            report.error(item_loc, "`required` must be a boolean")

        for str_field in ("description", "defaultValue", "placeholder", "validation"):
            if str_field in item and item[str_field] is not None and not _is_str(item[str_field]):
                report.error(item_loc, f"`{str_field}` must be a string")

        # SELECT / MULTI_SELECT / RADIO need `options`.
        if type_ in {"SELECT", "MULTI_SELECT", "RADIO"}:
            options = item.get("options")
            if not isinstance(options, list) or not options:
                report.error(item_loc, f"`options` is required for type {type_}")
            elif not all(_is_str(o) for o in options):
                report.error(item_loc, f"`options` for type {type_} must be a list of strings; the app parses ModuleConfigItem.options as List<String>, so object forms like [{{\"value\",\"label\"}}] fail Gson deserialization")


# ───────────────────────── registry checks ─────────────────────────────

def _validate_registry_entry(
    report: Report, entry: dict[str, Any], index: int
) -> None:
    where = f"registry.json::modules[{index}]"

    for required in ("id", "path", "name"):
        if not _is_str(entry.get(required)) or not entry[required].strip():
            report.error(where, f"`{required}` is required and must be a non-empty string")

    path = entry.get("path", "")
    if _is_str(path) and not KEBAB_CASE_RE.match(path):
        report.error(where, f"`path` must be kebab-case, got {path!r}")

    version = entry.get("version")
    if not _is_str(version) or not SEMVER_RE.match(version or ""):
        report.error(where, f"`version` must be a semver string, got {version!r}")

    category = entry.get("category", "OTHER")
    if not _is_str(category) or category not in ALLOWED_CATEGORIES:
        report.error(where, f"`category` must be one of the allowed values, got {category!r}")

    run_at = entry.get("runAt", "DOCUMENT_END")
    if not _is_str(run_at) or run_at not in ALLOWED_RUN_AT:
        report.error(where, f"`runAt` must be one of the allowed values, got {run_at!r}")

    for perm in _expect_list_of_str(report, where, entry.get("permissions"), "permissions"):
        if perm not in ALLOWED_PERMISSIONS:
            report.error(where, f"unknown permission {perm!r}")

    _validate_url_matches(report, where, entry.get("urlMatches"))
    _validate_author(report, where, entry.get("author"))

    if "minAppVersion" in entry and not isinstance(entry["minAppVersion"], int):
        report.error(where, "`minAppVersion` must be an integer")

    if "hasCss" in entry and not isinstance(entry["hasCss"], bool):
        report.error(where, "`hasCss` must be a boolean")

    icon_url = entry.get("iconUrl")
    if icon_url is not None and not _is_str(icon_url):
        report.error(where, "`iconUrl` must be a string when present")

    source_type = entry.get("sourceType", "CUSTOM")
    if not _is_str(source_type) or source_type not in ALLOWED_SOURCE_TYPES:
        report.error(where, f"`sourceType` must be one of {sorted(ALLOWED_SOURCE_TYPES)}, got {source_type!r}")

    if source_type == "CHROME_EXTENSION":
        store_id = entry.get("storeId")
        if not _is_str(store_id) or not STORE_ID_RE.match(store_id or ""):
            report.error(where, "`storeId` is required for CHROME_EXTENSION and must be a 32-char Chrome Web Store ID")


def _validate_registry(report: Report, registry: dict[str, Any]) -> list[dict[str, Any]]:
    where = "registry.json"
    schema = registry.get("schema")
    if schema != 1:
        report.error(where, f"`schema` must be 1 (got {schema!r}); future versions need a parser update")
    if "updatedAt" in registry and not _is_str(registry["updatedAt"]):
        report.error(where, "`updatedAt` must be a string")

    modules = registry.get("modules")
    if not isinstance(modules, list):
        report.error(where, "`modules` must be a list")
        return []

    seen_ids: set[str] = set()
    seen_paths: set[str] = set()
    for i, entry in enumerate(modules):
        if not isinstance(entry, dict):
            report.error(f"{where}::modules[{i}]", "must be an object")
            continue
        _validate_registry_entry(report, entry, i)

        eid = entry.get("id") if _is_str(entry.get("id")) else None
        if eid:
            if eid in seen_ids:
                report.error(f"{where}::modules[{i}]", f"duplicate id {eid!r}")
            seen_ids.add(eid)

        epath = entry.get("path") if _is_str(entry.get("path")) else None
        if epath:
            if epath in seen_paths:
                report.error(f"{where}::modules[{i}]", f"duplicate path {epath!r}")
            seen_paths.add(epath)

    return [m for m in modules if isinstance(m, dict)]


# ───────────────────────── cross-file consistency ─────────────────────

def _validate_cross_consistency(
    report: Report,
    entry: dict[str, Any],
    manifest: dict[str, Any],
    folder: Path,
) -> None:
    """`module.json` and `registry.json` must agree on the shared fields."""

    where = f"modules/{folder.name}"

    # id
    if entry.get("id") != manifest.get("id"):
        report.error(
            where,
            f"`id` mismatch: registry says {entry.get('id')!r}, manifest says {manifest.get('id')!r}",
        )

    # name
    if entry.get("name") != manifest.get("name"):
        report.error(
            where,
            f"`name` mismatch: registry says {entry.get('name')!r}, manifest says {manifest.get('name')!r}",
        )

    # version: registry stores the string, manifest stores an object.
    reg_version = entry.get("version")
    man_version_obj = manifest.get("version") or {}
    man_version = man_version_obj.get("name") if isinstance(man_version_obj, dict) else None
    if reg_version != man_version:
        report.error(
            where,
            f"`version` mismatch: registry={reg_version!r}, module.json::version.name={man_version!r}",
        )

    # runAt
    if entry.get("runAt") and manifest.get("runAt") and entry["runAt"] != manifest["runAt"]:
        report.error(
            where,
            f"`runAt` mismatch: registry={entry['runAt']!r}, manifest={manifest['runAt']!r}",
        )

    # permissions: registry should be a superset (the listing surface) of manifest.
    reg_perms = set(entry.get("permissions") or [])
    man_perms = set(manifest.get("permissions") or [])
    missing_in_registry = man_perms - reg_perms
    if missing_in_registry:
        report.error(
            where,
            f"manifest declares permissions {sorted(missing_in_registry)} that registry.json does not list",
        )

    # hasCss must match style.css presence on disk.
    has_css_flag = bool(entry.get("hasCss"))
    style_present = (folder / "style.css").is_file()
    if has_css_flag and not style_present:
        report.error(
            where,
            "registry says `hasCss: true` but no `style.css` file is present",
        )
    if style_present and not has_css_flag:
        report.error(
            where,
            "`style.css` exists but registry has `hasCss: false` — it will not be downloaded",
        )


# ───────────────────────── per-folder structural checks ───────────────

def _validate_folder_layout(report: Report, folder: Path) -> None:
    """Each module folder must look like the schema says it does."""

    where = f"modules/{folder.name}"

    if not KEBAB_CASE_RE.match(folder.name):
        report.error(where, f"folder name must be kebab-case, got {folder.name!r}")

    if not (folder / "module.json").is_file():
        report.error(where, "missing required `module.json`")

    if not (folder / "main.js").is_file():
        report.error(where, "missing required `main.js`")

    # Flag stray files. The runtime ignores them, so they only bloat the repo.
    expected = {"module.json", "main.js", "style.css"}
    for child in folder.iterdir():
        if child.name in expected or child.name in ALLOWED_EXTRA_FILES:
            continue
        if child.name in ALLOWED_ICON_FILES:
            # Icons are size-checked separately further below.
            continue
        if child.is_dir():
            report.warning(
                where,
                f"unexpected sub-directory `{child.name}/` — the runtime won't read it",
            )
        else:
            report.warning(
                where,
                f"unexpected file `{child.name}` — the runtime won't download it",
            )


def _validate_icon_coherence(
    report: Report, folder: Path, entry: dict[str, Any] | None
) -> None:
    """
    Cross-check the contributor-supplied icon against the registry.

    The contract:

    - If `iconUrl` is missing/empty, the app falls back to the first letter
      of the module name. No icon file is needed.
    - If `iconUrl` looks like a relative path, the corresponding file must
      actually exist next to `main.js` and stay within the size budget.
    - If `iconUrl` is absolute (starts with `http://` or `https://`), we
      defer to the contributor's hosting and do not check the file system.
    - The file name must come from `ALLOWED_ICON_FILES` so we don't end up
      shipping random binaries through the catalog.
    """
    where = f"modules/{folder.name}"
    icon_url = (entry or {}).get("iconUrl")

    icon_files_present = sorted(
        f for f in ALLOWED_ICON_FILES if (folder / f).is_file()
    )

    # Warn about extra random icon files when the registry doesn't reference
    # them — the runtime won't fetch a `logo.png` it has no way to know about.
    if not icon_url and icon_files_present:
        report.warning(
            where,
            f"icon file(s) {icon_files_present} present but registry has no "
            "`iconUrl` — they won't be used by the runtime",
        )

    if not _is_str(icon_url) or not icon_url.strip():
        return

    icon_url = icon_url.strip()
    if icon_url.startswith("http://") or icon_url.startswith("https://"):
        if icon_url.startswith("http://"):
            report.warning(where, "`iconUrl` uses http://; prefer https for catalog assets")
        return

    # Relative path case. Reject path traversal and force the file to live
    # inside the module folder with a known extension.
    normalised = icon_url.lstrip("./")
    if normalised.startswith("/") or ".." in normalised.split("/"):
        report.error(where, f"`iconUrl` must be a path inside the module folder, got {icon_url!r}")
        return
    if normalised not in ALLOWED_ICON_FILES:
        report.error(
            where,
            f"`iconUrl` must reference one of {sorted(ALLOWED_ICON_FILES)}, got {icon_url!r}",
        )
        return

    icon_path = folder / normalised
    if not icon_path.is_file():
        report.error(where, f"`iconUrl` points at {normalised!r} but no such file exists")
        return

    size = icon_path.stat().st_size
    if size > MAX_ICON_BYTES:
        report.error(
            where,
            f"`{normalised}` is {size // 1024} KB, over the {MAX_ICON_BYTES // 1024} KB cap; "
            "trim the icon or host it off-repo via an absolute URL",
        )


def _validate_main_js(report: Report, folder: Path) -> None:
    """Cheap heuristics on `main.js`."""

    main_js = folder / "main.js"
    if not main_js.is_file():
        return
    where = f"modules/{folder.name}/main.js"

    try:
        content = main_js.read_text(encoding="utf-8")
    except UnicodeDecodeError:
        report.error(where, "must be UTF-8")
        return

    if not content.strip():
        report.error(where, "is empty")
        return

    if len(content) > 512 * 1024:
        report.warning(where, f"is large ({len(content) // 1024} KB) — consider trimming")

    # No top-level `return` — the IIFE wrapper turns this into a syntax error.
    for lineno, line in enumerate(content.splitlines(), start=1):
        stripped = line.strip()
        if stripped.startswith("return ") or stripped == "return;" or stripped == "return":
            # Could still be inside a function literal at line 1; we can't
            # parse JS exactly, but this is a strong smell.
            indent = len(line) - len(line.lstrip())
            if indent == 0:
                report.error(
                    where,
                    f"line {lineno}: top-level `return` — the IIFE wrapper would make this a syntax error",
                )
                break

    # If `getConfig` is referenced but no configItems are declared in the
    # manifest, the user will see "undefined" defaults silently.
    manifest_path = folder / "module.json"
    if manifest_path.is_file() and "getConfig(" in content:
        try:
            manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
            if not manifest.get("configItems"):
                report.warning(
                    where,
                    "uses `getConfig(...)` but `module.json` declares no `configItems`",
                )
        except (json.JSONDecodeError, OSError):
            # The JSON-level checks elsewhere will report the actual cause.
            pass


# ───────────────────────── entry point ─────────────────────────────────

def _load_json(report: Report, path: Path, where: str) -> dict[str, Any] | None:
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except FileNotFoundError:
        report.error(where, f"file not found: {path}")
    except json.JSONDecodeError as e:
        report.error(where, f"invalid JSON: {e.msg} (line {e.lineno}, col {e.colno})")
    except UnicodeDecodeError:
        report.error(where, "file must be UTF-8")
    return None


def main(repo_root: Path) -> int:
    modules_dir = repo_root / "modules"
    if not modules_dir.is_dir():
        print(f"❌ {modules_dir} does not exist", file=sys.stderr)
        return 1

    report = Report()

    # 1. Parse + validate the registry.
    registry = _load_json(report, modules_dir / "registry.json", "registry.json")
    registry_entries: list[dict[str, Any]] = []
    if isinstance(registry, dict):
        registry_entries = _validate_registry(report, registry)

    # 2. Walk module folders.
    folders = sorted(
        p for p in modules_dir.iterdir()
        if p.is_dir() and not p.name.startswith(".")
    )

    folder_paths = {p.name for p in folders}
    registry_paths = {e["path"] for e in registry_entries if _is_str(e.get("path"))}
    custom_paths = {e["path"] for e in registry_entries if _is_str(e.get("path")) and e.get("sourceType", "CUSTOM") != "CHROME_EXTENSION"}

    # Folders missing a registry entry.
    for orphan in folder_paths - registry_paths:
        report.error(f"modules/{orphan}", "module folder has no entry in registry.json")
    # Registry entries pointing at non-existent folders (CHROME_EXTENSION entries don't need folders).
    for ghost in custom_paths - folder_paths:
        report.error("registry.json", f"entry refers to missing folder modules/{ghost}/")

    # 3. Per-folder validation + cross-file consistency.
    entries_by_path = {e["path"]: e for e in registry_entries if _is_str(e.get("path")) and e.get("sourceType", "CUSTOM") != "CHROME_EXTENSION"}
    for folder in folders:
        _validate_folder_layout(report, folder)
        _validate_main_js(report, folder)
        _validate_icon_coherence(report, folder, entries_by_path.get(folder.name))

        manifest_path = folder / "module.json"
        manifest = _load_json(report, manifest_path, f"modules/{folder.name}/module.json")
        if isinstance(manifest, dict):
            _validate_module_json(report, folder, manifest)
            entry = entries_by_path.get(folder.name)
            if entry:
                _validate_cross_consistency(report, entry, manifest, folder)

    print(report.render())
    return 0 if report.ok() else 1


if __name__ == "__main__":
    repo = Path(__file__).resolve().parents[3]
    sys.exit(main(repo))