ray-project--ray/python/ray/setup-dev.py

#!/usr/bin/env python
# ruff: noqa: E402
"""This script allows you to develop Ray Python code without needing to compile
Ray.
See https://docs.ray.io/en/master/development.html#building-ray-python-only"""

import os
import sys

# types.py can conflict with stdlib's types.py in some python versions,
# see https://github.com/python/cpython/issues/101210.
# To avoid import errors, we move the current working dir to the end of sys.path.
this_dir = os.path.dirname(__file__)
if this_dir in sys.path:
    sys.path.remove(this_dir)
    sys.path.append(this_dir)

import argparse
import shutil
import subprocess

import click

import ray


def do_link(package, force=False, skip_list=None, allow_list=None, local_path=None):
    if skip_list and package in skip_list:
        print(f"Skip creating symbolic link for {package}")
        return
    if allow_list is not None and package not in allow_list:
        print(f"Skip creating symbolic link for {package} (not in allow list)")
        return
    package_home = os.path.abspath(os.path.join(ray.__file__, f"../{package}"))
    # Infer local_path automatically.
    if local_path is None:
        local_path = f"../{package}"
    local_home = os.path.abspath(os.path.join(__file__, local_path))
    # If installed package dir does not exist, continue either way. We'll
    # remove it/create a link from there anyways.
    if not os.path.isdir(package_home) and not os.path.isfile(package_home):
        print(f"{package_home} does not exist. Continuing to link.")
    # Make sure the path we are linking to does exist.
    assert os.path.exists(local_home), local_home
    # Confirm with user.
    if not force and not click.confirm(
        f"This will replace:\n  {package_home}\nwith " f"a symlink to:\n  {local_home}",
        default=True,
    ):
        return

    # Windows: Create directory junction.
    if os.name == "nt":
        try:
            shutil.rmtree(package_home)
        except FileNotFoundError:
            pass
        except OSError:
            os.remove(package_home)

        # create symlink for directory or file
        if os.path.isdir(local_home):
            subprocess.check_call(
                ["mklink", "/J", package_home, local_home], shell=True
            )
        elif os.path.isfile(local_home):
            subprocess.check_call(
                ["mklink", "/H", package_home, local_home], shell=True
            )
        else:
            print(f"{local_home} is neither directory nor file. Link failed.")

    # Posix: Use `ln -s` to create softlink.
    else:
        if os.path.exists(package_home) and os.path.realpath(
            package_home
        ) == os.path.realpath(local_home):
            print(f"{package} is already linked to source. Skipping.")
            return
        sudo = []
        if not os.access(os.path.dirname(package_home), os.W_OK):
            print("You don't have write permission " f"to {package_home}, using sudo:")
            sudo = ["sudo"]
        print(f"Creating symbolic link from \n {local_home} to \n {package_home}")

        # Preserve ray/serve/generated
        serve_temp_dir = "/tmp/ray/_serve/"
        if package == "serve":
            # Copy generated folder to a temp dir
            generated_folder = os.path.join(package_home, "generated")
            if os.path.exists(serve_temp_dir):
                subprocess.check_call(sudo + ["rm", "-rf", serve_temp_dir])
            subprocess.check_call(sudo + ["mkdir", "-p", serve_temp_dir])
            subprocess.check_call(sudo + ["mv", generated_folder, serve_temp_dir])

        # Create backup of the old directory if it exists
        if os.path.exists(package_home):
            backup_dir = f"{package_home}.bak"
            print(f"Creating backup of {package_home} to {backup_dir}")
            subprocess.check_call(sudo + ["cp", "-r", package_home, backup_dir])

        subprocess.check_call(sudo + ["rm", "-rf", package_home])
        subprocess.check_call(sudo + ["ln", "-s", local_home, package_home])

        # Move generated folder to local_home
        if package == "serve":
            tmp_generated_folder = os.path.join(serve_temp_dir, "generated")
            package_generated_folder = os.path.join(package_home, "generated")
            if not os.path.exists(package_generated_folder):
                subprocess.check_call(
                    ["mv", tmp_generated_folder, package_generated_folder]
                )


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter, description="Setup dev."
    )
    parser.add_argument(
        "--yes", "-y", action="store_true", help="Don't ask for confirmation."
    )
    parser.add_argument(
        "--skip",
        "-s",
        nargs="*",
        help="List of folders to skip linking to facilitate workspace dev",
        required=False,
    )
    parser.add_argument(
        "--allow",
        "-a",
        nargs="*",
        help="List of folders to link (only these will be linked)",
        required=False,
    )
    parser.add_argument(
        "--extras",
        "-e",
        nargs="*",
        help="List of extra folders to link to facilitate workspace dev",
        required=False,
    )

    args = parser.parse_args()
    if args.skip and args.allow:
        print("Error: --skip and --allow cannot be used together.")
        sys.exit(1)

    if not args.yes:
        print("NOTE: Use '-y' to override all python files without confirmation.")

    # Dictionary of packages to link, with optional local_path
    packages_to_link = {
        "llm": None,
        "serve/llm": None,
        "data/llm.py": None,
        "rllib": "../../../rllib",
        "air": None,
        "tune": None,
        "train": None,
        "autoscaler": None,
        "cloudpickle": None,
        "data": None,
        "scripts": None,
        "internal": None,
        "tests": None,
        "experimental": None,
        "util": None,
        "workflow": None,
        "serve": None,
        "dag": None,
        "widgets": None,
        "cluster_utils.py": None,
        "_private": None,
        "_common": None,
        "dashboard": None,
    }

    # Link all packages using a for loop
    for package, local_path in packages_to_link.items():
        do_link(
            package,
            force=args.yes,
            skip_list=args.skip,
            allow_list=args.allow,
            local_path=local_path,
        )

    if args.extras is not None:
        for package in args.extras:
            do_link(package, force=args.yes, skip_list=args.skip, allow_list=args.allow)

    print(
        "Created links.\n\nIf you run into issues initializing Ray, please "
        "ensure that your local repo and the installed Ray are in sync "
        "(pip install -U the latest wheels at "
        "https://docs.ray.io/en/master/installation.html, "
        "and ensure you are up-to-date on the master branch on git).\n\n"
        "Note that you may need to delete the package symlinks when pip "
        "installing new Ray versions to prevent pip from overwriting files "
        "in your git repo."
    )