wehub-resource-sync
e93507a09c
Lockfile supply-chain audit / lockfile supply-chain audit (push) Has been cancelled
Windows Studio GGUF CI / GPU prebuilt resolves without Visual Studio (push) Has been cancelled
Windows Studio GGUF CI / setup.ps1 unit tests (VS 2026 / CMake guard) (push) Has been cancelled
Windows Studio GGUF CI / real-VS detection (VS 2022) (push) Has been cancelled
Windows Studio GGUF CI / real-VS detection (VS 2026) (push) Has been cancelled
Windows Studio GGUF CI / VC++ runtime detect + install round-trip (windows-2025-vs2026) (push) Has been cancelled
Windows Studio GGUF CI / VC++ runtime detect + install round-trip (windows-latest) (push) Has been cancelled
Windows Studio Update CI / Studio Updating Tests (push) Has been cancelled
Wheel CI / Wheel build + content sanity + import smoke (push) Has been cancelled
Lint CI / Source lint (Python + shell + YAML + JSON + safety nets) (push) Has been cancelled
MLX CI on Mac M1 / dispatch (push) Has been cancelled
Security audit / advisory audit (pip + npm + cargo) (push) Has been cancelled
Security audit / pip scan-packages :: extras (push) Has been cancelled
Security audit / pip scan-packages :: studio (push) Has been cancelled
Security audit / pip scan-packages :: hf-stack (push) Has been cancelled
Security audit / npm scan-packages (Studio frontend tarballs) (push) Has been cancelled
Security audit / workflow-trigger lint (pull_request_target / cache-poisoning) (push) Has been cancelled
Security audit / pytest tests/security (push) Has been cancelled
Security audit / npm provenance + new install-script diff (push) Has been cancelled
Studio API CI / Studio API & Auth Tests (push) Has been cancelled
Backend CI / (Python 3.10) (push) Has been cancelled
Backend CI / (Python 3.11) (push) Has been cancelled
Backend CI / (Python 3.12) (push) Has been cancelled
Backend CI / (Python 3.13) (push) Has been cancelled
Backend CI / Repo tests (CPU) (push) Has been cancelled
Frontend CI / Frontend build + bundle sanity (push) Has been cancelled
Studio GGUF CI / OpenAI, Anthropic API tests (push) Has been cancelled
Studio GGUF CI / Tool calling Tests (push) Has been cancelled
Studio GGUF CI / JSON, images (push) Has been cancelled
Mac Studio GGUF CI / OpenAI, Anthropic API tests (push) Has been cancelled
Mac Studio GGUF CI / Tool calling Tests (push) Has been cancelled
Mac Studio GGUF CI / JSON, images (push) Has been cancelled
Mac Studio Install Matrix CI / Install + load (macos-14) (push) Has been cancelled
Mac Studio Install Matrix CI / Install + load (macos-15) (push) Has been cancelled
Mac Studio Install Matrix CI / Install + load (macos-26) (push) Has been cancelled
Mac Studio Install Matrix CI / Install + load (macos-15-intel) (push) Has been cancelled
Mac Studio API CI / Studio API & Auth Tests (push) Has been cancelled
Mac Studio Install Matrix CI / Install + load (macos-26-intel) (push) Has been cancelled
Mac Studio UI CI / Chat UI Tests (push) Has been cancelled
Studio Tauri CI / Tauri Linux debug build (no codesign) (push) Has been cancelled
Mac Studio Update CI / Studio Updating Tests (push) Has been cancelled
Studio UI CI / Chat UI Tests (push) Has been cancelled
Windows Studio API CI / Studio API & Auth Tests (push) Has been cancelled
Windows Studio UI CI / Chat UI Tests (push) Has been cancelled
Studio Update CI / Studio Updating Tests (push) Has been cancelled
Core / Core (HF=default + TRL=default) (push) Has been cancelled
Core / Core (HF=4.57.6 + TRL<1) (push) Has been cancelled
Core / Core (HF=latest + TRL=latest) (push) Has been cancelled
Core / llama.cpp build + smoke (push) Has been cancelled
Windows Studio GGUF CI / OpenAI, Anthropic API tests (push) Has been cancelled
Windows Studio GGUF CI / Tool calling Tests (push) Has been cancelled
Windows Studio GGUF CI / JSON, images (push) Has been cancelled
Windows Studio GGUF CI / Studio install + inference without Visual Studio (push) Has been cancelled
Studio export capability / capability (macos-latest) (push) Has been cancelled
Studio export capability / capability (ubuntu-latest) (push) Has been cancelled
Studio export capability / capability (windows-latest) (push) Has been cancelled
Cross-platform parity / parity (macos-latest) (push) Has been cancelled
Cross-platform parity / parity (windows-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
Studio load-orchestrator CI / test (push) Has been cancelled
87 lines
2.2 KiB
Python
87 lines
2.2 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""Dataset utilities for LLM/VLM fine-tuning: detection, conversion, templating, collators, mappings."""
|
|
|
|
from .format_detection import (
|
|
detect_dataset_format,
|
|
detect_custom_format_heuristic,
|
|
detect_multimodal_dataset,
|
|
detect_vlm_dataset_structure,
|
|
)
|
|
|
|
from .format_conversion import (
|
|
standardize_chat_format,
|
|
convert_chatml_to_alpaca,
|
|
convert_alpaca_to_chatml,
|
|
convert_to_vlm_format,
|
|
convert_llava_to_vlm_format,
|
|
convert_sharegpt_with_images_to_vlm_format,
|
|
)
|
|
|
|
from .chat_templates import (
|
|
apply_chat_template_to_dataset,
|
|
get_dataset_info_summary,
|
|
get_tokenizer_chat_template,
|
|
DEFAULT_ALPACA_TEMPLATE,
|
|
)
|
|
|
|
from .vlm_processing import (
|
|
generate_smart_vlm_instruction,
|
|
)
|
|
|
|
from .data_collators import (
|
|
DataCollatorSpeechSeq2SeqWithPadding,
|
|
DeepSeekOCRDataCollator,
|
|
VLMDataCollator,
|
|
)
|
|
|
|
from .model_mappings import (
|
|
TEMPLATE_TO_MODEL_MAPPER,
|
|
MODEL_TO_TEMPLATE_MAPPER,
|
|
TEMPLATE_TO_RESPONSES_MAPPER,
|
|
is_gpt_oss_model_name,
|
|
)
|
|
|
|
# Legacy dataset_utils.py imports kept for backward compat
|
|
from .dataset_utils import (
|
|
check_dataset_format,
|
|
format_and_template_dataset,
|
|
format_dataset,
|
|
)
|
|
|
|
__all__ = [
|
|
# Detection
|
|
"detect_dataset_format",
|
|
"detect_custom_format_heuristic",
|
|
"detect_multimodal_dataset",
|
|
"detect_vlm_dataset_structure",
|
|
# Conversion
|
|
"standardize_chat_format",
|
|
"convert_chatml_to_alpaca",
|
|
"convert_alpaca_to_chatml",
|
|
"convert_to_vlm_format",
|
|
"convert_llava_to_vlm_format",
|
|
"convert_sharegpt_with_images_to_vlm_format",
|
|
# Templates
|
|
"apply_chat_template_to_dataset",
|
|
"get_dataset_info_summary",
|
|
"get_tokenizer_chat_template",
|
|
"DEFAULT_ALPACA_TEMPLATE",
|
|
# VLM
|
|
"generate_smart_vlm_instruction",
|
|
# Collators
|
|
"DataCollatorSpeechSeq2SeqWithPadding",
|
|
"DeepSeekOCRDataCollator",
|
|
"VLMDataCollator",
|
|
# Mappings
|
|
"TEMPLATE_TO_MODEL_MAPPER",
|
|
"MODEL_TO_TEMPLATE_MAPPER",
|
|
"TEMPLATE_TO_RESPONSES_MAPPER",
|
|
"is_gpt_oss_model_name",
|
|
# Main entry points
|
|
"check_dataset_format",
|
|
"format_and_template_dataset",
|
|
"format_dataset",
|
|
]
|