wehub-resource-sync
87 lines
3.1 KiB
Python
87 lines
3.1 KiB
Python
class UnknownPreprocessorError(ValueError):
|
|
"""Raised when attempting to deserialize an unknown/unregistered preprocessor type."""
|
|
|
|
def __init__(self, preprocessor_type: str):
|
|
self.preprocessor_type = preprocessor_type
|
|
super().__init__(f"Unknown preprocessor type: {preprocessor_type}")
|
|
|
|
|
|
_PREPROCESSOR_REGISTRY = {}
|
|
|
|
|
|
def SerializablePreprocessor(version: int, identifier: str):
|
|
"""Register a preprocessor class for serialization.
|
|
|
|
This decorator registers a preprocessor class in the serialization registry,
|
|
enabling it to be serialized and deserialized. The decorated class MUST inherit
|
|
from SerializablePreprocessor.
|
|
|
|
Args:
|
|
version: Version number for this preprocessor's serialization format
|
|
identifier: Stable identifier for serialization. This identifier will be used
|
|
in serialized data. Using an explicit identifier allows classes to be
|
|
renamed without breaking compatibility with existing serialized data.
|
|
|
|
Returns:
|
|
A decorator function that registers the class and returns it unchanged.
|
|
|
|
Raises:
|
|
TypeError: If the decorated class does not inherit from SerializablePreprocessor
|
|
|
|
Note:
|
|
If a class with the same identifier is already registered, logs a debug message
|
|
and overwrites the previous registration.
|
|
|
|
Examples:
|
|
@SerializablePreprocessor(version=1, identifier="my_preprocessor_v1")
|
|
class MyPreprocessor(SerializablePreprocessor):
|
|
pass
|
|
"""
|
|
|
|
def decorator(cls):
|
|
import logging
|
|
|
|
from ray.data.preprocessor import SerializablePreprocessorBase
|
|
|
|
# Verify that the class inherits from SerializablePreprocessor
|
|
if not issubclass(cls, SerializablePreprocessorBase):
|
|
raise TypeError(
|
|
f"Class {cls.__module__}.{cls.__qualname__} must inherit from "
|
|
f"SerializablePreprocessor to use @SerializablePreprocessor decorator."
|
|
)
|
|
|
|
cls.set_version(version)
|
|
cls.set_preprocessor_class_id(identifier)
|
|
|
|
# Check for collisions and log debug message
|
|
if identifier in _PREPROCESSOR_REGISTRY:
|
|
existing = _PREPROCESSOR_REGISTRY[identifier]
|
|
if existing != cls:
|
|
logging.debug(
|
|
f"Preprocessor id collision: '{identifier}' was already registered "
|
|
f"by {existing.__module__}.{existing.__qualname__}. "
|
|
f"Overwriting with {cls.__module__}.{cls.__qualname__}."
|
|
)
|
|
|
|
_PREPROCESSOR_REGISTRY[identifier] = cls
|
|
return cls
|
|
|
|
return decorator
|
|
|
|
|
|
def _lookup_class(serialization_id: str):
|
|
"""Look up a preprocessor class by its serialization ID.
|
|
|
|
Args:
|
|
serialization_id: The serialization ID of the preprocessor (either explicit or class name)
|
|
|
|
Returns:
|
|
The registered preprocessor class
|
|
|
|
Raises:
|
|
UnknownPreprocessorError: If the serialization ID is not registered
|
|
"""
|
|
if serialization_id not in _PREPROCESSOR_REGISTRY:
|
|
raise UnknownPreprocessorError(serialization_id)
|
|
return _PREPROCESSOR_REGISTRY[serialization_id]
|