chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,297 @@
|
||||
"""A utility for debugging serialization issues."""
|
||||
|
||||
import inspect
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Optional, Set, Tuple
|
||||
|
||||
import colorama
|
||||
|
||||
# Import ray first to use the bundled colorama
|
||||
import ray # noqa: F401
|
||||
import ray.cloudpickle as cp
|
||||
from ray.util.annotations import DeveloperAPI
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _indent(printer):
|
||||
printer.level += 1
|
||||
yield
|
||||
printer.level -= 1
|
||||
|
||||
|
||||
class _Printer:
|
||||
def __init__(self, print_file):
|
||||
self.level = 0
|
||||
self.print_file = print_file
|
||||
|
||||
def indent(self):
|
||||
return _indent(self)
|
||||
|
||||
def print(self, msg):
|
||||
indent = " " * self.level
|
||||
print(indent + msg, file=self.print_file)
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
class FailureTuple:
|
||||
"""Represents the serialization 'frame'.
|
||||
|
||||
Attributes:
|
||||
obj: The object that fails serialization.
|
||||
path: Tuple of variable names representing the traversal path.
|
||||
parent: The object that references the `obj`.
|
||||
"""
|
||||
|
||||
def __init__(self, obj: Any, path: tuple[str, ...], parent: Any):
|
||||
"""Initialize FailureTuple.
|
||||
|
||||
Args:
|
||||
obj: The object that fails serialization.
|
||||
path: Tuple of variable names representing the traversal path.
|
||||
parent: The object that references the object.
|
||||
"""
|
||||
self.obj = obj
|
||||
self.path = path
|
||||
self.parent = parent
|
||||
|
||||
def __repr__(self):
|
||||
formatted_path = [str(p) if p is not None else "unknown" for p in self.path]
|
||||
path_str = " -> ".join(formatted_path)
|
||||
var_name = formatted_path[-1] if formatted_path else "unknown"
|
||||
return f"FailTuple({var_name} [path={path_str}, obj={self.obj}, parent={self.parent}])"
|
||||
|
||||
|
||||
def _inspect_func_serialization(base_obj, depth, parent, failure_set, printer, path=()):
|
||||
"""Adds the first-found non-serializable element to the failure_set."""
|
||||
assert inspect.isfunction(base_obj)
|
||||
printer.print(f"Inspecting closure of '{base_obj.__qualname__}':")
|
||||
closure = inspect.getclosurevars(base_obj)
|
||||
found = False
|
||||
if closure.globals:
|
||||
printer.print(
|
||||
f"Detected {len(closure.globals)} global variables. "
|
||||
"Checking serializability..."
|
||||
)
|
||||
|
||||
with printer.indent():
|
||||
for name, obj in closure.globals.items():
|
||||
serializable, _ = _inspect_serializability(
|
||||
obj,
|
||||
name=name,
|
||||
depth=depth - 1,
|
||||
parent=parent,
|
||||
failure_set=failure_set,
|
||||
printer=printer,
|
||||
path=path,
|
||||
)
|
||||
found = found or not serializable
|
||||
if found:
|
||||
break
|
||||
|
||||
if closure.nonlocals:
|
||||
printer.print(
|
||||
f"Detected {len(closure.nonlocals)} nonlocal variables. "
|
||||
"Checking serializability..."
|
||||
)
|
||||
with printer.indent():
|
||||
for name, obj in closure.nonlocals.items():
|
||||
serializable, _ = _inspect_serializability(
|
||||
obj,
|
||||
name=name,
|
||||
depth=depth - 1,
|
||||
parent=parent,
|
||||
failure_set=failure_set,
|
||||
printer=printer,
|
||||
path=path,
|
||||
)
|
||||
found = found or not serializable
|
||||
if found:
|
||||
break
|
||||
if not found:
|
||||
obj_name = getattr(base_obj, "__qualname__", None) or repr(base_obj)
|
||||
printer.print(
|
||||
f"WARNING: Did not find non-serializable object in {obj_name}. "
|
||||
"This may be because the object is only non-serializable when "
|
||||
"combined with other objects, or because cloudpickle and the "
|
||||
"traversal disagree on which object causes the failure. "
|
||||
"Try calling `ray.util.inspect_serializability` directly on "
|
||||
"the suspected object, or simplify the closure to isolate the cause."
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
def _inspect_generic_serialization(
|
||||
base_obj, depth, parent, failure_set, printer, path=()
|
||||
):
|
||||
"""Adds the first-found non-serializable element to the failure_set."""
|
||||
assert not inspect.isfunction(base_obj)
|
||||
functions = inspect.getmembers(base_obj, predicate=inspect.isfunction)
|
||||
found = False
|
||||
with printer.indent():
|
||||
for name, obj in functions:
|
||||
serializable, _ = _inspect_serializability(
|
||||
obj,
|
||||
name=name,
|
||||
depth=depth - 1,
|
||||
parent=parent,
|
||||
failure_set=failure_set,
|
||||
printer=printer,
|
||||
path=path,
|
||||
)
|
||||
found = found or not serializable
|
||||
if found:
|
||||
break
|
||||
|
||||
with printer.indent():
|
||||
members = inspect.getmembers(base_obj)
|
||||
for name, obj in members:
|
||||
if name.startswith("__") and name.endswith("__") or inspect.isbuiltin(obj):
|
||||
continue
|
||||
serializable, _ = _inspect_serializability(
|
||||
obj,
|
||||
name=name,
|
||||
depth=depth - 1,
|
||||
parent=parent,
|
||||
failure_set=failure_set,
|
||||
printer=printer,
|
||||
path=path,
|
||||
)
|
||||
found = found or not serializable
|
||||
if found:
|
||||
break
|
||||
if not found:
|
||||
obj_name = (
|
||||
base_obj.__name__ if inspect.isclass(base_obj) else type(base_obj).__name__
|
||||
)
|
||||
printer.print(
|
||||
f"WARNING: Did not find non-serializable object in "
|
||||
f"{obj_name if inspect.isclass(base_obj) else f'{obj_name} instance'}. "
|
||||
"This may be because the object is only non-serializable when "
|
||||
"combined with other objects, or because cloudpickle and the "
|
||||
"traversal disagree. "
|
||||
"Try calling `ray.util.inspect_serializability` on individual "
|
||||
"attributes to isolate the cause."
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
def inspect_serializability(
|
||||
base_obj: Any,
|
||||
name: Optional[str] = None,
|
||||
depth: int = 3,
|
||||
print_file: Optional[Any] = None,
|
||||
) -> Tuple[bool, Set[FailureTuple]]:
|
||||
"""Identifies what objects are preventing serialization.
|
||||
|
||||
Args:
|
||||
base_obj: Object to be serialized.
|
||||
name: Optional name of string.
|
||||
depth: Depth of the scope stack to walk through. Defaults to 3.
|
||||
print_file: file argument that will be passed to print().
|
||||
|
||||
Returns:
|
||||
bool: True if serializable.
|
||||
set[FailureTuple]: Set of unserializable objects.
|
||||
|
||||
.. versionadded:: 1.1.0
|
||||
|
||||
"""
|
||||
printer = _Printer(print_file)
|
||||
return _inspect_serializability(base_obj, name, depth, None, None, printer)
|
||||
|
||||
|
||||
def _inspect_serializability(
|
||||
base_obj, name, depth, parent, failure_set, printer, path=()
|
||||
) -> Tuple[bool, Set[FailureTuple]]:
|
||||
colorama.init()
|
||||
top_level = False
|
||||
declaration = ""
|
||||
found = False
|
||||
if failure_set is None:
|
||||
top_level = True
|
||||
failure_set = set()
|
||||
declaration = f"Checking Serializability of {base_obj}"
|
||||
printer.print("=" * min(len(declaration), 80))
|
||||
printer.print(declaration)
|
||||
printer.print("=" * min(len(declaration), 80))
|
||||
else:
|
||||
printer.print(f"Serializing '{name}' {base_obj}...")
|
||||
try:
|
||||
cp.dumps(base_obj)
|
||||
return True, failure_set
|
||||
except Exception as e:
|
||||
printer.print(
|
||||
f"{colorama.Fore.RED}!!! FAIL{colorama.Fore.RESET} " f"serialization: {e}"
|
||||
)
|
||||
found = True
|
||||
try:
|
||||
if depth == 0:
|
||||
failure_path = path + ((name,) if name is not None else ())
|
||||
failure_set.add(FailureTuple(base_obj, failure_path, parent))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if depth <= 0:
|
||||
return False, failure_set
|
||||
|
||||
# TODO: we only differentiate between 'function' and 'object'
|
||||
# but we should do a better job of diving into something
|
||||
# more specific like a Type, Object, etc.
|
||||
if inspect.isfunction(base_obj):
|
||||
_inspect_func_serialization(
|
||||
base_obj,
|
||||
depth=depth,
|
||||
parent=base_obj,
|
||||
failure_set=failure_set,
|
||||
printer=printer,
|
||||
path=path + ((name,) if name is not None else ()),
|
||||
)
|
||||
else:
|
||||
_inspect_generic_serialization(
|
||||
base_obj,
|
||||
depth=depth,
|
||||
parent=base_obj,
|
||||
failure_set=failure_set,
|
||||
printer=printer,
|
||||
path=path + ((name,) if name is not None else ()),
|
||||
)
|
||||
|
||||
if not failure_set:
|
||||
failure_path = path + ((name,) if name is not None else ())
|
||||
failure_set.add(FailureTuple(base_obj, failure_path, parent))
|
||||
|
||||
if top_level:
|
||||
printer.print("=" * min(len(declaration), 80))
|
||||
if not failure_set:
|
||||
printer.print(
|
||||
"Nothing failed the inspect_serialization test, though "
|
||||
"serialization did not succeed."
|
||||
)
|
||||
else:
|
||||
fail_vars = (
|
||||
f"\n\n\t{colorama.Style.BRIGHT}"
|
||||
+ "\n".join(str(k) for k in failure_set)
|
||||
+ f"{colorama.Style.RESET_ALL}\n\n"
|
||||
)
|
||||
printer.print(
|
||||
f"Variable: {fail_vars}was found to be non-serializable. "
|
||||
"There may be multiple other undetected variables that were "
|
||||
"non-serializable. "
|
||||
)
|
||||
printer.print(
|
||||
"Consider either removing the "
|
||||
"instantiation/imports of these variables or moving the "
|
||||
"instantiation into the scope of the function/class. "
|
||||
)
|
||||
printer.print("=" * min(len(declaration), 80))
|
||||
printer.print(
|
||||
"Check https://docs.ray.io/en/master/ray-core/objects/serialization.html#troubleshooting for more information." # noqa
|
||||
)
|
||||
printer.print(
|
||||
"If you have any suggestions on how to improve "
|
||||
"this error message, please reach out to the "
|
||||
"Ray developers on github.com/ray-project/ray/issues/"
|
||||
)
|
||||
printer.print("=" * min(len(declaration), 80))
|
||||
return not found, failure_set
|
||||
Reference in New Issue
Block a user