wehub-resource-sync
2114ccd278
CI / Lint & Test (Python 3.13) (push) Failing after 2s
CI / Lint & Test (Python 3.14) (push) Failing after 1s
CI / Lint & Test (Python 3.12) (push) Failing after 2s
CI / DCO Check (push) Has been skipped
Scorecard supply-chain security / Scorecard analysis (push) Failing after 2s
92 lines
3.1 KiB
Python
92 lines
3.1 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Language detection via Unicode script ratio analysis.
|
|
|
|
Zero external dependencies — uses only the standard-library ``unicodedata``
|
|
module, the same one the main SkillSpector project already imports in
|
|
``mcp_tool_poisoning.py``.
|
|
|
|
Approach: count CJK / Hiragana / Katakana / Hangul characters against
|
|
total alphabetic content. A configurable ratio threshold decides the
|
|
dominant language. This avoids heavyweight ML-based detectors while
|
|
being accurate enough for the batch-scan use case.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unicodedata
|
|
|
|
# Unicode range constants — (start, end) inclusive.
|
|
_CJK_UNIFIED = (0x4E00, 0x9FFF) # CJK Unified Ideographs
|
|
_CJK_EXT_A = (0x3400, 0x4DBF) # CJK Unified Ideographs Extension A
|
|
_HIRAGANA = (0x3040, 0x309F)
|
|
_KATAKANA = (0x30A0, 0x30FF)
|
|
_HANGUL = (0xAC00, 0xD7AF) # Hangul Syllables
|
|
|
|
# Thresholds — a skill file is classified as non-English when the ratio of
|
|
# CJK / kana / Hangul characters exceeds this proportion of total alpha chars.
|
|
_CJK_THRESHOLD = 0.10
|
|
_KANA_THRESHOLD = 0.05
|
|
_HANGUL_THRESHOLD = 0.10
|
|
|
|
|
|
def _in_range(cp: int, r: tuple[int, int]) -> bool:
|
|
return r[0] <= cp <= r[1]
|
|
|
|
|
|
def detect_language(content: str) -> str:
|
|
"""Heuristic single-file language detection.
|
|
|
|
Returns one of ``"zh"``, ``"ja"``, ``"ko"``, or ``"en"``.
|
|
"""
|
|
cjk = kana = hangul = alpha = 0
|
|
for ch in content:
|
|
cp = ord(ch)
|
|
if _in_range(cp, _CJK_UNIFIED) or _in_range(cp, _CJK_EXT_A):
|
|
cjk += 1
|
|
elif _in_range(cp, _HIRAGANA) or _in_range(cp, _KATAKANA):
|
|
kana += 1
|
|
elif _in_range(cp, _HANGUL):
|
|
hangul += 1
|
|
if unicodedata.category(ch).startswith("L"):
|
|
alpha += 1
|
|
|
|
if alpha == 0:
|
|
return "en"
|
|
|
|
if kana / alpha > _KANA_THRESHOLD:
|
|
return "ja"
|
|
if hangul / alpha > _HANGUL_THRESHOLD:
|
|
return "ko"
|
|
if cjk / alpha > _CJK_THRESHOLD:
|
|
return "zh"
|
|
return "en"
|
|
|
|
|
|
def detect_skill_language(file_cache: dict[str, str]) -> str:
|
|
"""Determine the dominant language across all files in a skill.
|
|
|
|
Aggregates per-file :func:`detect_language` results via majority vote.
|
|
When no non-English script is detected in any file, returns ``"en"``.
|
|
"""
|
|
votes: dict[str, int] = {}
|
|
for content in file_cache.values():
|
|
lang = detect_language(content)
|
|
votes[lang] = votes.get(lang, 0) + 1
|
|
if not votes:
|
|
return "en"
|
|
return max(votes, key=lambda k: votes[k]) # type: ignore[no-any-return]
|