chore: import upstream snapshot with attribution

python/ray/cloudpickle/README.md

@@ -0,0 +1,7 @@
+
+## How to pull upstream changes into the vendored cloudpickle
+
+Right now, we will need to update the vendored cloudpickle manually. Here are the steps to do so:
+1. Overwrite the `cloudpickle.py` and `cloudpickle_fast.py` from the [upstream](https://github.com/cloudpipe/cloudpickle)
+2. Update the version in python/ray/cloudpickle/__init__.py
+3. Run performance benchmarks to validate no regression. (e.g. microbenchmark, scalability tests, etc.)

python/ray/cloudpickle/__init__.py

@@ -0,0 +1,47 @@
+from __future__ import absolute_import
+
+import os
+from pickle import PicklingError
+
+from ray.cloudpickle.cloudpickle import *  # noqa
+from ray.cloudpickle.cloudpickle_fast import CloudPickler, dumps, dump  # noqa
+
+
+# Conform to the convention used by python serialization libraries, which
+# expose their Pickler subclass at top-level under the  "Pickler" name.
+Pickler = CloudPickler
+
+__version__ = '3.1.2'
+
+
+def _warn_msg(obj, method, exc):
+    return (
+        f"{method}({str(obj)}) failed."
+        "\nTo check which non-serializable variables are captured "
+        "in scope, re-run the ray script with 'RAY_PICKLE_VERBOSE_DEBUG=1'.")
+
+
+def dump_debug(obj, *args, **kwargs):
+    try:
+        return dump(obj, *args, **kwargs)
+    except (TypeError, PicklingError) as exc:
+        if os.environ.get("RAY_PICKLE_VERBOSE_DEBUG"):
+            from ray.util.check_serialize import inspect_serializability
+            inspect_serializability(obj)
+            raise
+        else:
+            msg = _warn_msg(obj, "ray.cloudpickle.dump", exc)
+            raise type(exc)(msg)
+
+
+def dumps_debug(obj, *args, **kwargs):
+    try:
+        return dumps(obj, *args, **kwargs)
+    except (TypeError, PicklingError) as exc:
+        if os.environ.get("RAY_PICKLE_VERBOSE_DEBUG"):
+            from ray.util.check_serialize import inspect_serializability
+            inspect_serializability(obj)
+            raise
+        else:
+            msg = _warn_msg(obj, "ray.cloudpickle.dumps", exc)
+            raise type(exc)(msg)

python/ray/cloudpickle/cloudpickle_fast.py

@@ -0,0 +1,14 @@
+"""Compatibility module.
+
+It can be necessary to load files generated by previous versions of cloudpickle
+that rely on symbols being defined under the `cloudpickle.cloudpickle_fast`
+namespace.
+
+See: tests/test_backward_compat.py
+"""
+
+from . import cloudpickle
+
+
+def __getattr__(name):
+    return getattr(cloudpickle, name)

python/ray/cloudpickle/compat.py

@@ -0,0 +1,19 @@
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+RAY_PICKLE_VERBOSE_DEBUG = os.environ.get("RAY_PICKLE_VERBOSE_DEBUG")
+verbose_level = int(RAY_PICKLE_VERBOSE_DEBUG) if RAY_PICKLE_VERBOSE_DEBUG else 0
+
+if verbose_level > 1:
+    logger.warning(
+        "Environmental variable RAY_PICKLE_VERBOSE_DEBUG is set to "
+        f"'{verbose_level}', this enabled python-based serialization backend "
+        f"instead of C-Pickle. Serialization would be very slow."
+    )
+    from ray.cloudpickle import py_pickle as pickle
+    from ray.cloudpickle.py_pickle import Pickler
+else:
+    import pickle  # noqa: F401
+    from _pickle import Pickler  # noqa: F401

python/ray/cloudpickle/py_pickle.py

@@ -0,0 +1,30 @@
+from pickle import (
+    _Pickler,
+    _Unpickler as Unpickler,
+    _loads as loads,
+    _load as load,
+    PickleError,
+    PicklingError,
+    UnpicklingError,
+    HIGHEST_PROTOCOL,
+)
+
+__all__ = [
+    "PickleError",
+    "PicklingError",
+    "UnpicklingError",
+    "Pickler",
+    "Unpickler",
+    "load",
+    "loads",
+    "HIGHEST_PROTOCOL",
+]
+
+
+class Pickler(_Pickler):
+    def __init__(self, file, protocol=None, *, fix_imports=True, buffer_callback=None):
+        super().__init__(
+            file, protocol, fix_imports=fix_imports, buffer_callback=buffer_callback
+        )
+        # avoid being overrided by cloudpickle
+        self.dispatch = _Pickler.dispatch.copy()