ray-project--ray/python/ray/_private/runtime_env/rocprof_sys.py

import asyncio
import copy
import logging
import os
import subprocess
import sys
from pathlib import Path
from typing import Dict, List, Optional, Tuple

from ray._common.utils import try_to_create_directory
from ray._private.runtime_env.context import RuntimeEnvContext
from ray._private.runtime_env.plugin import RuntimeEnvPlugin
from ray.exceptions import RuntimeEnvSetupError

default_logger = logging.getLogger(__name__)

# rocprof-sys config used when runtime_env={"_rocprof_sys": "default"}
# Refer to the following link for more information on rocprof-sys options
# https://rocm.docs.amd.com/projects/rocprofiler-systems/en/docs-6.4.0/how-to/understanding-rocprof-sys-output.html
ROCPROFSYS_DEFAULT_CONFIG = {
    "env": {
        "ROCPROFSYS_TIME_OUTPUT": "false",
        "ROCPROFSYS_OUTPUT_PREFIX": "worker_process_%p",
    },
    "args": {
        "F": "true",
    },
}


def parse_rocprof_sys_config(
    rocprof_sys_config: Dict[str, str]
) -> Tuple[List[str], List[str]]:
    """
    Function to convert dictionary of rocprof-sys options into
    rocprof-sys-python command line

    The function returns:
    - List[str]: rocprof-sys-python cmd line split into list of str
    """
    rocprof_sys_cmd = ["rocprof-sys-python"]
    rocprof_sys_env = {}
    if "args" in rocprof_sys_config:
        # Parse rocprof-sys arg options
        for option, option_val in rocprof_sys_config["args"].items():
            # option standard based on
            # https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html
            if len(option) > 1:
                rocprof_sys_cmd.append(f"--{option}={option_val}")
            else:
                rocprof_sys_cmd += [f"-{option}", option_val]
    if "env" in rocprof_sys_config:
        rocprof_sys_env = rocprof_sys_config["env"]
    rocprof_sys_cmd.append("--")
    return rocprof_sys_cmd, rocprof_sys_env


class RocProfSysPlugin(RuntimeEnvPlugin):
    name = "_rocprof_sys"

    def __init__(self, resources_dir: str):
        self.rocprof_sys_cmd = []
        self.rocprof_sys_env = {}

        # replace this with better way to get logs dir
        session_dir, runtime_dir = os.path.split(resources_dir)
        self._rocprof_sys_dir = Path(session_dir) / "logs" / "rocprof_sys"
        try_to_create_directory(self._rocprof_sys_dir)

    async def _check_rocprof_sys_script(
        self, rocprof_sys_config: Dict[str, str]
    ) -> Tuple[bool, str]:
        """
        Function to validate if rocprof_sys_config is a valid rocprof_sys profile options
        Args:
            rocprof_sys_config: dictionary mapping rocprof_sys option to it's value
        Returns:
            a tuple consists of a boolean indicating if the rocprof_sys_config
            is valid option and an error message if the rocprof_sys_config is invalid
        """

        # use empty as rocprof_sys report test filename
        test_folder = str(Path(self._rocprof_sys_dir) / "test")
        rocprof_sys_cmd, rocprof_sys_env = parse_rocprof_sys_config(rocprof_sys_config)
        rocprof_sys_env_copy = copy.deepcopy(rocprof_sys_env)
        rocprof_sys_env_copy["ROCPROFSYS_OUTPUT_PATH"] = test_folder
        rocprof_sys_env_copy.update(os.environ)
        try_to_create_directory(test_folder)

        # Create a test python file to run rocprof_sys
        with open(f"{test_folder}/test.py", "w") as f:
            f.write("import time\n")
        try:
            rocprof_sys_cmd = rocprof_sys_cmd + [f"{test_folder}/test.py"]
            process = await asyncio.create_subprocess_exec(
                *rocprof_sys_cmd,
                env=rocprof_sys_env_copy,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            stdout, stderr = await process.communicate()
            error_msg = stderr.strip() if stderr.strip() != "" else stdout.strip()

            # cleanup temp file
            clean_up_cmd = ["rm", "-r", test_folder]
            cleanup_process = await asyncio.create_subprocess_exec(
                *clean_up_cmd,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            _, _ = await cleanup_process.communicate()
            if process.returncode == 0:
                return True, None
            else:
                return False, error_msg
        except FileNotFoundError:
            return False, ("rocprof_sys is not installed")

    async def create(
        self,
        uri: Optional[str],
        runtime_env: "RuntimeEnv",  # noqa: F821
        context: RuntimeEnvContext,
        logger: logging.Logger = default_logger,
    ) -> int:
        rocprof_sys_config = runtime_env.rocprof_sys()
        if not rocprof_sys_config:
            return 0

        if rocprof_sys_config and sys.platform != "linux":
            raise RuntimeEnvSetupError("rocprof-sys CLI is only available in Linux.\n")

        if isinstance(rocprof_sys_config, str):
            if rocprof_sys_config == "default":
                rocprof_sys_config = ROCPROFSYS_DEFAULT_CONFIG
            else:
                raise RuntimeEnvSetupError(
                    f"Unsupported rocprof_sys config: {rocprof_sys_config}. "
                    "The supported config is 'default' or "
                    "Dictionary of rocprof_sys options"
                )

        is_valid_rocprof_sys_config, error_msg = await self._check_rocprof_sys_script(
            rocprof_sys_config
        )
        if not is_valid_rocprof_sys_config:
            logger.warning(error_msg)
            raise RuntimeEnvSetupError(
                "rocprof-sys profile failed to run with the following "
                f"error message:\n {error_msg}"
            )
        # add set output path to logs dir
        if "env" not in rocprof_sys_config:
            rocprof_sys_config["env"] = {}
        rocprof_sys_config["env"]["ROCPROFSYS_OUTPUT_PATH"] = str(
            Path(self._rocprof_sys_dir)
        )

        self.rocprof_sys_cmd, self.rocprof_sys_env = parse_rocprof_sys_config(
            rocprof_sys_config
        )
        return 0

    def modify_context(
        self,
        uris: List[str],
        runtime_env: "RuntimeEnv",  # noqa: F821
        context: RuntimeEnvContext,
        logger: Optional[logging.Logger] = default_logger,
    ):
        logger.info("Running rocprof-sys profiler")
        context.py_executable = " ".join(self.rocprof_sys_cmd)
        context.env_vars.update(self.rocprof_sys_env)