wehub-resource-sync
497 lines
14 KiB
Python
497 lines
14 KiB
Python
# Bazel build
|
|
# C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html
|
|
|
|
# ************************** IMPORTANT ***********************
|
|
# If a target or file is not found here, then please try searching in /src
|
|
# We are in the middle of transitioning to a tree of build files.
|
|
# If you would like to help with the move in your PR, please use `git mv` so that the history of the file is retained.
|
|
|
|
load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
|
|
load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library")
|
|
load("@hedron_compile_commands//:refresh_compile_commands.bzl", "refresh_compile_commands")
|
|
load("@rules_cc//cc:defs.bzl", "cc_proto_library")
|
|
load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files")
|
|
load("@rules_pkg//pkg:zip.bzl", "pkg_zip")
|
|
load("@rules_proto//proto:defs.bzl", "proto_library")
|
|
load("@rules_python//python:defs.bzl", "py_binary", "py_library")
|
|
load("//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS", "ray_cc_library")
|
|
|
|
package(
|
|
default_visibility = ["//visibility:public"],
|
|
)
|
|
|
|
exports_files([
|
|
".rayciversion",
|
|
"ray-images.json",
|
|
])
|
|
|
|
# C/C++ toolchain constraint configs.
|
|
|
|
config_setting(
|
|
name = "msvc-cl",
|
|
flag_values = {"@bazel_tools//tools/cpp:compiler": "msvc-cl"},
|
|
)
|
|
|
|
config_setting(
|
|
name = "clang-cl",
|
|
flag_values = {"@bazel_tools//tools/cpp:compiler": "clang-cl"},
|
|
)
|
|
|
|
config_setting(
|
|
name = "opt",
|
|
values = {"compilation_mode": "opt"},
|
|
)
|
|
|
|
bool_flag(
|
|
name = "jemalloc_flag",
|
|
build_setting_default = True,
|
|
)
|
|
|
|
config_setting(
|
|
name = "jemalloc",
|
|
constraint_values = ["@platforms//os:linux"],
|
|
flag_values = {":jemalloc_flag": "true"},
|
|
)
|
|
|
|
alias(
|
|
name = "uv_file",
|
|
actual = select({
|
|
"//bazel:linux_x86_64_config": "@uv_x86_64-linux//:file",
|
|
"//bazel:osx_arm64_config": "@uv_aarch64-darwin//:file",
|
|
"//conditions:default": "@uv_x86_64-linux//:file",
|
|
}),
|
|
)
|
|
|
|
# bazel run :refresh_compile_commands for compile_commands generation for clangd
|
|
# https://github.com/hedronvision/bazel-compile-commands-extractor?tab=readme-ov-file#vscode - directions for clangd config
|
|
refresh_compile_commands(
|
|
name = "refresh_compile_commands",
|
|
exclude_external_sources = False,
|
|
targets = {
|
|
"//src/ray/...": "",
|
|
},
|
|
)
|
|
|
|
ray_cc_library(
|
|
name = "ray_mock",
|
|
# NOTE(edoakes): we are moving towards fine-grained mock and fake targets.
|
|
# Do not include new files in this target, instead make a BUILD.bazel file
|
|
# in the subdirectory and exclude it here.
|
|
hdrs = glob(
|
|
["src/mock/**/*.h"],
|
|
exclude = [
|
|
"src/mock/ray/common/pubsub/publisher.h",
|
|
"src/mock/ray/common/pubsub/subscriber.h",
|
|
"src/mock/ray/ray_syncer/ray_syncer.h",
|
|
],
|
|
),
|
|
deps = [
|
|
"//src/ray/asio:periodical_runner",
|
|
"//src/ray/observability:fake_metric",
|
|
"//src/ray/observability:fake_ray_event_recorder",
|
|
"//src/ray/pubsub:fake_publisher",
|
|
"//src/ray/util:clock",
|
|
],
|
|
)
|
|
|
|
# This header is used to wrap some internal codes so we can reduce suspicious
|
|
# symbols export.
|
|
ray_cc_library(
|
|
name = "exported_internal",
|
|
srcs =
|
|
[
|
|
"src/ray/internal/internal.cc",
|
|
],
|
|
hdrs =
|
|
[
|
|
"src/ray/internal/internal.h",
|
|
],
|
|
copts = COPTS,
|
|
strip_include_prefix = "src",
|
|
deps = [
|
|
"//src/ray/core_worker:core_worker_lib",
|
|
],
|
|
alwayslink = 1,
|
|
)
|
|
|
|
ray_cc_library(
|
|
name = "platform_shims",
|
|
srcs = [] + select({
|
|
"@platforms//os:windows": glob([
|
|
"src/shims/windows/**/*.c",
|
|
"src/shims/windows/**/*.cc",
|
|
"src/shims/windows/**/*.h",
|
|
]),
|
|
"//conditions:default": [],
|
|
}),
|
|
hdrs = [] + select({
|
|
"@platforms//os:windows": glob([
|
|
"src/shims/windows/**/*.h",
|
|
]),
|
|
"//conditions:default": [],
|
|
}),
|
|
strip_include_prefix = select({
|
|
"@platforms//os:windows": "src/shims/windows",
|
|
"//conditions:default": "",
|
|
}),
|
|
)
|
|
|
|
filegroup(
|
|
name = "extra_actions_base_proto",
|
|
srcs = [
|
|
"thirdparty/protobuf/extra_actions_base.proto",
|
|
],
|
|
)
|
|
|
|
proto_library(
|
|
name = "extra_actions_base_proto_lib",
|
|
srcs = ["thirdparty/protobuf/extra_actions_base.proto"],
|
|
)
|
|
|
|
cc_proto_library(
|
|
name = "extra_actions_cc_proto_lib",
|
|
deps = [":extra_actions_base_proto_lib"],
|
|
)
|
|
|
|
alias(
|
|
name = "hiredis",
|
|
actual = "@com_github_redis_hiredis//:hiredis",
|
|
)
|
|
|
|
pyx_library(
|
|
name = "_raylet",
|
|
srcs = glob([
|
|
"python/ray/includes/*.pxd",
|
|
"python/ray/includes/*.pxi",
|
|
]) + [
|
|
"python/ray/__init__.py",
|
|
"python/ray/_raylet.pxd",
|
|
"python/ray/_raylet.pyx",
|
|
],
|
|
# Export ray ABI symbols, which can then be used by _streaming.so.
|
|
# We need to dlopen this lib with RTLD_GLOBAL to use ABI in this
|
|
# shared lib, see python/ray/__init__.py.
|
|
cc_kwargs = dict(
|
|
srcs = PYX_SRCS,
|
|
# cython code is auto-generated, which is out of our control.
|
|
copts = COPTS + PYX_COPTS,
|
|
# This is needed since Windows has long file path issues and command length limits
|
|
features = ["compiler_param_file"],
|
|
# see https://github.com/tensorflow/tensorflow/blob/r2.1/tensorflow/lite/BUILD#L444
|
|
linkopts = select({
|
|
"@platforms//os:osx": [
|
|
"-Wl,-exported_symbols_list,$(location //:src/ray/ray_exported_symbols.lds)",
|
|
# The Apple CC toolchain adds -undefined dynamic_lookup implicitly via
|
|
# cc_toolchain_config.bzl for dynamic library link actions. Making it
|
|
# explicit here ensures Python extension .so files link correctly regardless
|
|
# of which CC toolchain is active (Apple CC or generic @local_config_cc).
|
|
"-Wl,-undefined,dynamic_lookup",
|
|
],
|
|
"@platforms//os:windows": [],
|
|
"//conditions:default": [
|
|
"-Wl,--version-script,$(location //:src/ray/ray_version_script.lds)",
|
|
],
|
|
}),
|
|
linkstatic = 1,
|
|
),
|
|
deps = [
|
|
"//:exported_internal",
|
|
"//:src/ray/ray_exported_symbols.lds",
|
|
"//:src/ray/ray_version_script.lds",
|
|
"//src/ray/common:python_callbacks",
|
|
"//src/ray/core_worker:core_worker_lib",
|
|
"//src/ray/gcs:gcs_server_lib",
|
|
"//src/ray/gcs/store_client:redis_store_client",
|
|
"//src/ray/gcs_rpc_client:global_state_accessor_lib",
|
|
"//src/ray/observability:python_event_interface",
|
|
"//src/ray/observability:ray_event_recorder_interface",
|
|
"//src/ray/protobuf:serialization_cc_proto",
|
|
"//src/ray/pubsub:python_gcs_subscriber",
|
|
"//src/ray/raylet_rpc_client:raylet_client_with_io_context_lib",
|
|
"//src/ray/thirdparty/setproctitle",
|
|
"//src/ray/util:memory",
|
|
"//src/ray/util:port_persistence",
|
|
"//src/ray/util:raii",
|
|
"//src/ray/util:stream_redirection",
|
|
"//src/ray/util:stream_redirection_options",
|
|
],
|
|
)
|
|
|
|
filegroup(
|
|
name = "python_sources",
|
|
srcs = glob([
|
|
"python/ray/*.py",
|
|
"python/ray/cloudpickle/*.py",
|
|
"python/ray/dashboard/**/*.py",
|
|
"python/ray/experimental/*.py",
|
|
"python/ray/util/*.py",
|
|
"python/ray/internal/*.py",
|
|
]) + [
|
|
"python/ray/core/__init__.py",
|
|
"//python/ray/_private:src_files",
|
|
"//python/ray/autoscaler:default_configs",
|
|
"//python/ray/autoscaler:src_files",
|
|
],
|
|
)
|
|
|
|
alias(
|
|
name = "redis-server",
|
|
actual = select({
|
|
"@platforms//os:windows": "@com_github_tporadowski_redis_bin//:redis-server.exe",
|
|
"//bazel:linux_x86_64_config": "@redis_linux_x86_64//:redis-server",
|
|
"//bazel:linux_arm64_config": "@redis_linux_arm64//:redis-server",
|
|
"//bazel:osx_arm64_config": "@redis_osx_arm64//:redis-server",
|
|
"//conditions:default": "@com_github_antirez_redis//:redis-server",
|
|
}),
|
|
)
|
|
|
|
alias(
|
|
name = "redis-cli",
|
|
actual = select({
|
|
"@platforms//os:windows": "@com_github_tporadowski_redis_bin//:redis-cli.exe",
|
|
"//bazel:linux_x86_64_config": "@redis_linux_x86_64//:redis-cli",
|
|
"//bazel:linux_arm64_config": "@redis_linux_arm64//:redis-cli",
|
|
"//bazel:osx_arm64_config": "@redis_osx_arm64//:redis-cli",
|
|
"//conditions:default": "@com_github_antirez_redis//:redis-cli",
|
|
}),
|
|
)
|
|
|
|
filegroup(
|
|
name = "core_py_proto",
|
|
srcs = ["//src/ray/protobuf:core_py_proto"],
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
filegroup(
|
|
name = "serve_py_proto",
|
|
srcs = ["//src/ray/protobuf:serve_py_proto"],
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
# This is a dummy test dependency that causes the python tests to be
|
|
# re-run if any of these files changes.
|
|
py_library(
|
|
name = "ray_lib",
|
|
srcs = glob(
|
|
["python/ray/**/*.py"],
|
|
exclude = ["python/ray/tests/*.py"],
|
|
),
|
|
visibility = ["__subpackages__"],
|
|
)
|
|
|
|
pkg_files(
|
|
name = "raylet_so_files",
|
|
srcs = ["python/ray/_raylet.so"],
|
|
attributes = pkg_attributes(mode = "755"),
|
|
prefix = "ray/",
|
|
renames = select({
|
|
"@platforms//os:windows": {
|
|
"python/ray/_raylet.so": "_raylet.pyd",
|
|
},
|
|
"//conditions:default": {},
|
|
}),
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
pkg_files(
|
|
name = "core_py_proto_files",
|
|
srcs = ["//src/ray/protobuf:core_py_proto"],
|
|
prefix = "ray/core/generated",
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
pkg_zip(
|
|
name = "core_py_proto_zip",
|
|
srcs = [":core_py_proto_files"],
|
|
out = "core_py_proto.zip",
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
pkg_files(
|
|
name = "serve_py_proto_files",
|
|
srcs = [":serve_py_proto"],
|
|
prefix = "ray/serve/generated",
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
pkg_zip(
|
|
name = "serve_py_proto_zip",
|
|
srcs = [":serve_py_proto_files"],
|
|
out = "serve_py_proto.zip",
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
pkg_files(
|
|
name = "redis_files",
|
|
srcs = [
|
|
":redis-cli",
|
|
":redis-server",
|
|
],
|
|
attributes = pkg_attributes(mode = "755"),
|
|
prefix = "ray/core/src/ray/thirdparty/redis/src",
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
pkg_files(
|
|
name = "raylet_files",
|
|
srcs = ["//src/ray/raylet"],
|
|
attributes = pkg_attributes(mode = "755"),
|
|
prefix = "ray/core/src/ray/raylet",
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
pkg_files(
|
|
name = "gcs_server_files",
|
|
srcs = ["//src/ray/gcs:gcs_server"],
|
|
attributes = pkg_attributes(mode = "755"),
|
|
prefix = "ray/core/src/ray/gcs",
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
pkg_files(
|
|
name = "jemalloc_files",
|
|
srcs = ["@jemalloc//:shared"],
|
|
attributes = pkg_attributes(mode = "755"),
|
|
prefix = "ray/core/",
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
pkg_zip(
|
|
name = "ray_pkg_zip",
|
|
srcs = [
|
|
":gcs_server_files",
|
|
":raylet_files",
|
|
":raylet_so_files",
|
|
] + select({
|
|
":jemalloc": [":jemalloc_files"],
|
|
"//conditions:default": [],
|
|
}),
|
|
out = "ray_pkg.zip",
|
|
visibility = ["//java:__pkg__"],
|
|
)
|
|
|
|
genrule(
|
|
name = "ray_py_proto_zip",
|
|
srcs = [
|
|
":core_py_proto_zip",
|
|
":serve_py_proto_zip",
|
|
],
|
|
outs = ["ray_py_proto.zip"],
|
|
cmd = """
|
|
set -euo pipefail
|
|
|
|
tmpdir=$$(mktemp -d)
|
|
|
|
unzip -o -q $(location :core_py_proto_zip) -d "$$tmpdir"
|
|
unzip -o -q $(location :serve_py_proto_zip) -d "$$tmpdir"
|
|
|
|
files=(
|
|
$$(ls "$$tmpdir"/ray/core/generated/*_pb2*.py)
|
|
$$(ls "$$tmpdir"/ray/serve/generated/*_pb2*.py)
|
|
)
|
|
|
|
# NOTE(hchen): Protobuf doesn't allow specifying Python package name. So we use this `sed`
|
|
# command to change the import path in the generated file.
|
|
sed -i -E 's/from src.ray.protobuf.public/from ./' "$${files[@]}"
|
|
sed -i -E 's/from src.ray.protobuf/from ./' "$${files[@]}"
|
|
# Help the generated serve files to have the correct module
|
|
serve_files=($$(ls "$$tmpdir"/ray/serve/generated/*_pb2*.py))
|
|
sed -i -E 's/'"'"'src.ray.protobuf./'"'"'ray.serve.generated./' "$${serve_files[@]}"
|
|
|
|
# TODO(sang): Build our own proto instead of creating a new proto for opencensus separately.
|
|
# https://github.com/ray-project/ray/issues/31358
|
|
sed -i -E 's/from opencensus.proto.metrics.v1 import/from . import/' "$${files[@]}"
|
|
sed -i -E 's/from opencensus.proto.resource.v1 import/from . import/' "$${files[@]}"
|
|
|
|
$(location //bazel:pyzip) "$$tmpdir" $@
|
|
|
|
rm -rf "$$tmpdir"
|
|
""",
|
|
tools = [
|
|
"//bazel:pyzip",
|
|
],
|
|
visibility = ["//visibility:private"],
|
|
)
|
|
|
|
genrule(
|
|
name = "install_py_proto",
|
|
srcs = [
|
|
":ray_py_proto_zip",
|
|
],
|
|
outs = ["install_py_proto.out"],
|
|
cmd = """
|
|
set -euo pipefail
|
|
|
|
rm -rf python/ray/core/generated python/ray/serve/generated
|
|
unzip -o -q $(location :ray_py_proto_zip) -d python
|
|
|
|
echo "$${PWD}" > $@
|
|
""",
|
|
local = 1,
|
|
)
|
|
|
|
genrule(
|
|
name = "ray_pkg",
|
|
srcs = [
|
|
":python_sources",
|
|
":ray_py_proto_zip",
|
|
":ray_pkg_zip",
|
|
],
|
|
outs = ["ray_pkg.out"],
|
|
cmd = """
|
|
set -euo pipefail
|
|
|
|
rm -rf python/ray/core/generated python/ray/serve/generated
|
|
unzip -o -q $(location :ray_py_proto_zip) -d "python"
|
|
unzip -o -q $(location :ray_pkg_zip) -d "python"
|
|
|
|
echo "$${PWD}" > $@
|
|
""",
|
|
local = 1,
|
|
)
|
|
|
|
pkg_zip(
|
|
name = "ray_redis_zip",
|
|
srcs = [
|
|
":redis_files",
|
|
],
|
|
out = "ray_redis.zip",
|
|
)
|
|
|
|
py_binary(
|
|
name = "gen_redis_pkg",
|
|
srcs = ["gen_redis_pkg.py"],
|
|
data = [
|
|
":ray_redis_zip",
|
|
],
|
|
visibility = ["//visibility:private"],
|
|
deps = [
|
|
"//bazel:gen_extract",
|
|
],
|
|
)
|
|
|
|
py_binary(
|
|
name = "gen_py_proto",
|
|
srcs = ["gen_py_proto.py"],
|
|
data = [
|
|
":ray_py_proto_zip",
|
|
],
|
|
visibility = ["//visibility:private"],
|
|
deps = [
|
|
"//bazel:gen_extract",
|
|
],
|
|
)
|
|
|
|
py_binary(
|
|
name = "gen_ray_pkg",
|
|
srcs = ["gen_ray_pkg.py"],
|
|
data = [
|
|
":ray_pkg_zip",
|
|
":ray_py_proto_zip",
|
|
],
|
|
visibility = ["//visibility:private"],
|
|
deps = [
|
|
"//bazel:gen_extract",
|
|
],
|
|
)
|