wehub-resource-sync
326 lines
12 KiB
Python
326 lines
12 KiB
Python
"""
|
|
Sets up software rasterizers for CI.
|
|
|
|
Borrows heavily from wgpu's CI setup.
|
|
See https://github.com/gfx-rs/wgpu/blob/a8a91737b2d2f378976e292074c75817593a0224/.github/workflows/ci.yml#L10
|
|
In fact we're the exact same Mesa builds that wgpu produces,
|
|
see https://github.com/gfx-rs/ci-build
|
|
|
|
Using a software rasterizer avoids GPU-related flakiness on CI runners which we hit quite often when
|
|
running many tests in parallel - we got spurious timeouts and failure to find graphics devices,
|
|
the cause of these issues is unknown. See https://github.com/rerun-io/rerun/issues/11359.
|
|
|
|
All platforms use lavapipe (Mesa's software Vulkan implementation).
|
|
For macOS, we use a custom static build from https://github.com/rerun-io/lavapipe-build.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from shutil import copytree
|
|
|
|
# Sourced from https://archive.mesa3d.org/. Bumping this requires
|
|
# updating the mesa build in https://github.com/gfx-rs/ci-build and creating a new release.
|
|
MESA_VERSION = "25.2.7"
|
|
|
|
# Corresponds to https://github.com/gfx-rs/ci-build/releases
|
|
CI_BINARY_BUILD = "build26"
|
|
|
|
# Lavapipe build for macOS from https://github.com/rerun-io/lavapipe-build
|
|
LAVAPIPE_MACOS_VERSION = "v0.4.0"
|
|
LAVAPIPE_MACOS_URL = (
|
|
f"https://github.com/rerun-io/lavapipe-build/releases/download/{LAVAPIPE_MACOS_VERSION}/lavapipe-macos-arm64.tar.gz"
|
|
)
|
|
|
|
CARGO_TARGET_DIR = Path(os.environ.get("CARGO_TARGET_DIR", "target"))
|
|
|
|
|
|
def run(
|
|
args: list[str],
|
|
*,
|
|
env: dict[str, str] | None = None,
|
|
timeout: int | None = None,
|
|
cwd: str | None = None,
|
|
retries: int = 0,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
for attempt in range(retries + 1):
|
|
result = subprocess.run(args, env=env, cwd=cwd, timeout=timeout, check=False, capture_output=True, text=True)
|
|
if result.returncode == 0:
|
|
return result
|
|
|
|
if attempt < retries:
|
|
print(
|
|
f"Command {args[0]} failed with exit-code {result.returncode}. Retrying in 5 seconds ({attempt + 1}/{retries})…"
|
|
)
|
|
time.sleep(5)
|
|
|
|
assert result.returncode == 0, (
|
|
f"{subprocess.list2cmdline(args)} failed with exit-code {result.returncode}. Output:\n{result.stdout}\n{result.stderr}"
|
|
)
|
|
return result
|
|
|
|
|
|
def set_environment_variables(variables: dict[str, str]) -> None:
|
|
"""
|
|
Sets environment variables in the GITHUB_ENV file.
|
|
|
|
If `GITHUB_ENV` is not set (i.e. when running locally), prints the variables to stdout.
|
|
"""
|
|
for key, value in variables.items():
|
|
os.environ[key] = value
|
|
|
|
# Set in GITHUB_ENV file.
|
|
github_env = os.environ.get("GITHUB_ENV")
|
|
if github_env is None:
|
|
print(f"GITHUB_ENV is not set. The following environment variables need to be set:\n{variables}")
|
|
else:
|
|
print(f"Setting environment variables in {github_env}:\n{variables}")
|
|
# Write to GITHUB_ENV file.
|
|
with open(github_env, "a", encoding="utf-8") as f:
|
|
for key, value in variables.items():
|
|
f.write(f"{key}={value}\n")
|
|
|
|
|
|
def setup_lavapipe_for_linux() -> dict[str, str]:
|
|
"""Sets up lavapipe mesa driver for Linux (x64)."""
|
|
# Download mesa
|
|
run(
|
|
[
|
|
"curl",
|
|
"-L",
|
|
"--retry",
|
|
"5",
|
|
f"https://github.com/gfx-rs/ci-build/releases/download/{CI_BINARY_BUILD}/mesa-{MESA_VERSION}-linux-x86_64.tar.xz",
|
|
"-o",
|
|
"mesa.tar.xz",
|
|
],
|
|
retries=3,
|
|
)
|
|
|
|
# Create mesa directory and extract
|
|
os.makedirs("mesa", exist_ok=True)
|
|
run(["tar", "xpf", "mesa.tar.xz", "-C", "mesa"])
|
|
|
|
# The ICD provided by the mesa build is hardcoded to the build environment.
|
|
# We write out our own ICD file to point to the mesa vulkan
|
|
icd_json = f"""{{
|
|
"ICD": {{
|
|
"api_version": "1.1.255",
|
|
"library_path": "{os.getcwd()}/mesa/lib/x86_64-linux-gnu/libvulkan_lvp.so"
|
|
}},
|
|
"file_format_version": "1.0.0"
|
|
}}"""
|
|
icd_json_path = Path("icd.json")
|
|
with open(icd_json_path, "w", encoding="utf-8") as f:
|
|
f.write(icd_json)
|
|
|
|
# Update environment variables
|
|
env_vars = {
|
|
"VK_DRIVER_FILES": f"{os.getcwd()}/{icd_json_path}",
|
|
"LD_LIBRARY_PATH": f"{os.getcwd()}/mesa/lib/x86_64-linux-gnu/:{os.environ.get('LD_LIBRARY_PATH', '')}",
|
|
}
|
|
set_environment_variables(env_vars)
|
|
|
|
# On CI we run with elevated privileges, therefore VK_DRIVER_FILES is ignored.
|
|
# See: https://github.com/KhronosGroup/Vulkan-Loader/blob/sdk-1.3.261/docs/LoaderInterfaceArchitecture.md#elevated-privilege-caveats
|
|
# (curiously, when installing the Vulkan SDK via apt, it seems to work fine).
|
|
# Therefore, we copy the icd file into one of the standard search paths.
|
|
target_path = Path("~/.config/vulkan/icd.d").expanduser()
|
|
print(f"Copying icd file to {target_path}")
|
|
target_path.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy(icd_json_path, target_path)
|
|
|
|
return env_vars
|
|
|
|
|
|
def setup_lavapipe_for_windows() -> dict[str, str]:
|
|
"""Sets up lavapipe mesa driver for Windows (x64)."""
|
|
|
|
# Download mesa
|
|
run(
|
|
[
|
|
"curl.exe",
|
|
"-L",
|
|
"--retry",
|
|
"5",
|
|
f"https://github.com/pal1000/mesa-dist-win/releases/download/{MESA_VERSION}/mesa3d-{MESA_VERSION}-release-msvc.7z",
|
|
"-o",
|
|
"mesa.7z",
|
|
],
|
|
retries=3,
|
|
)
|
|
|
|
# Extract needed files
|
|
run([
|
|
"7z.exe",
|
|
"e",
|
|
"mesa.7z",
|
|
"-aoa",
|
|
"-omesa",
|
|
"x64/vulkan_lvp.dll",
|
|
"x64/lvp_icd.x86_64.json",
|
|
])
|
|
|
|
# Copy files to target directory.
|
|
copytree("mesa", CARGO_TARGET_DIR / "debug", dirs_exist_ok=True)
|
|
copytree("mesa", CARGO_TARGET_DIR / "debug" / "deps", dirs_exist_ok=True)
|
|
|
|
# Print icd file that should be used.
|
|
icd_json_path = Path(os.path.join(os.getcwd(), "mesa", "lvp_icd.x86_64.json")).resolve()
|
|
print(f"Using ICD file at '{icd_json_path}':")
|
|
with open(icd_json_path, encoding="utf-8") as f:
|
|
print(f.read())
|
|
icd_json_path_str = icd_json_path.as_posix().replace("/", "\\")
|
|
|
|
# Set environment variables, make sure to use windows path style.
|
|
vulkan_runtime_path = f"{os.environ['VULKAN_SDK']}/runtime/x64"
|
|
env_vars = {
|
|
"VK_DRIVER_FILES": icd_json_path_str,
|
|
# Vulkan runtime install should do this, but the CI action we're using right now for instance doesn't,
|
|
# causing `vulkaninfo` to fail since it can't find the vulkan loader.
|
|
"PATH": f"{os.environ.get('PATH', '')};{vulkan_runtime_path}",
|
|
}
|
|
set_environment_variables(env_vars)
|
|
|
|
# For debugging: List files in Vulkan runtime path.
|
|
if False:
|
|
print(f"\nListing files in Vulkan runtime path '{vulkan_runtime_path}':")
|
|
try:
|
|
files = os.listdir(vulkan_runtime_path)
|
|
for file in files:
|
|
print(f" {file}")
|
|
except Exception as e:
|
|
print(f"Error listing Vulkan runtime directory: {e}")
|
|
|
|
# On CI we run with elevated privileges, therefore VK_DRIVER_FILES is ignored.
|
|
# See: https://github.com/KhronosGroup/Vulkan-Loader/blob/vulkan-sdk-1.4.304/docs/LoaderInterfaceArchitecture.md#elevated-privilege-caveats
|
|
# Therefore, we have to set one of the registry keys that is checked to find the driver.
|
|
# See: https://vulkan.lunarg.com/doc/view/1.4.304.1/windows/LoaderDriverInterface.html#driver-discovery-on-windows
|
|
|
|
# Write registry keys to configure Vulkan drivers
|
|
import winreg
|
|
|
|
key_path = "SOFTWARE\\Khronos\\Vulkan\\Drivers"
|
|
|
|
# mypy apparently doesn't have stubs for wingreg
|
|
key = winreg.CreateKeyEx(winreg.HKEY_LOCAL_MACHINE, key_path) # type: ignore[attr-defined]
|
|
winreg.SetValueEx(key, icd_json_path_str, 0, winreg.REG_DWORD, 0) # type: ignore[attr-defined]
|
|
winreg.CloseKey(key) # type: ignore[attr-defined]
|
|
|
|
return env_vars
|
|
|
|
|
|
def setup_lavapipe_for_macos() -> dict[str, str]:
|
|
"""Sets up lavapipe mesa driver for macOS (arm64)."""
|
|
print("Setting up lavapipe for macOS…")
|
|
|
|
# Download lavapipe build (self-contained with bundled dependencies)
|
|
run([
|
|
"curl",
|
|
"-L",
|
|
"--retry",
|
|
"5",
|
|
LAVAPIPE_MACOS_URL,
|
|
"-o",
|
|
"lavapipe-macos-arm64.tar.gz",
|
|
])
|
|
|
|
# Extract
|
|
run(["tar", "xzf", "lavapipe-macos-arm64.tar.gz"])
|
|
|
|
lavapipe_dir = Path(os.getcwd()) / "lavapipe-macos-arm64"
|
|
icd_json_path = lavapipe_dir / "lvp_icd.aarch64.json"
|
|
|
|
# The Vulkan SDK's loader may be incompatible with lavapipe. Replace it with
|
|
# Homebrew's vulkan-loader which is built against the same Vulkan headers as Mesa.
|
|
print("Installing Homebrew vulkan-loader for compatibility with lavapipe…")
|
|
run(["/opt/homebrew/bin/brew", "install", "vulkan-loader"])
|
|
|
|
# Replace the Vulkan SDK's loader with the Homebrew one in all search paths.
|
|
# DYLD_LIBRARY_PATH alone is unreliable on macOS due to SIP stripping it.
|
|
run(["sudo", "mkdir", "-p", "/usr/local/lib"])
|
|
for lib_name in ["libvulkan.dylib", "libvulkan.1.dylib"]:
|
|
homebrew_path = f"/opt/homebrew/lib/{lib_name}"
|
|
if Path(homebrew_path).exists():
|
|
run(["sudo", "ln", "-sf", homebrew_path, f"/usr/local/lib/{lib_name}"])
|
|
# Also overwrite in the Vulkan SDK lib directory
|
|
vulkan_sdk_path = os.environ.get("VULKAN_SDK")
|
|
if vulkan_sdk_path:
|
|
sdk_lib = f"{vulkan_sdk_path}/lib/{lib_name}"
|
|
if Path(sdk_lib).exists() or Path(f"{vulkan_sdk_path}/lib").is_dir():
|
|
print(f"Replacing {sdk_lib} with symlink to {homebrew_path}")
|
|
run(["sudo", "ln", "-sf", homebrew_path, sdk_lib])
|
|
|
|
# Set environment variables
|
|
env_vars = {
|
|
"VK_DRIVER_FILES": str(icd_json_path.resolve()),
|
|
# Clear VK_LAYER_PATH to prevent validation layer crashes with lavapipe.
|
|
"VK_LAYER_PATH": "",
|
|
}
|
|
set_environment_variables(env_vars)
|
|
|
|
return env_vars
|
|
|
|
|
|
def vulkan_info(extra_env_vars: dict[str, str]) -> None:
|
|
"""Run vulkaninfo to verify the Vulkan setup."""
|
|
vulkan_sdk_path = os.environ["VULKAN_SDK"]
|
|
|
|
env = os.environ.copy()
|
|
env["VK_LOADER_DEBUG"] = "all" # Enable verbose logging of vulkan loader for debugging.
|
|
for key, value in extra_env_vars.items():
|
|
env[key] = value
|
|
|
|
if os.name == "nt":
|
|
vulkaninfo_path = f"{vulkan_sdk_path}/bin/vulkaninfoSDK.exe"
|
|
else:
|
|
vulkaninfo_path = f"{vulkan_sdk_path}/bin/vulkaninfo"
|
|
|
|
if not Path(vulkaninfo_path).exists():
|
|
print(f"ERROR: vulkaninfo not found at {vulkaninfo_path}")
|
|
print("The Vulkan SDK should be installed with vulkaninfo utility.")
|
|
sys.exit(1)
|
|
|
|
print(f"\nRunning vulkaninfo to verify setup (from {vulkaninfo_path})…\n")
|
|
print(run([vulkaninfo_path, "--summary"], env=env).stdout)
|
|
|
|
|
|
def check_for_vulkan_sdk() -> None:
|
|
vulkan_sdk_path = os.environ.get("VULKAN_SDK")
|
|
if vulkan_sdk_path is None:
|
|
print(
|
|
"ERROR: VULKAN_SDK is not set. The sdk needs to be installed prior including runtime & vulkaninfo utility.",
|
|
)
|
|
sys.exit(1)
|
|
|
|
|
|
def main() -> None:
|
|
# We only use Vulkan software rasterizers right now.
|
|
check_for_vulkan_sdk()
|
|
|
|
if os.name == "nt" and platform.machine() == "AMD64":
|
|
# Note that we could also use WARP, the DX12 software rasterizer.
|
|
# (wgpu tests with both llvmpip and WARP)
|
|
# But practically speaking we prefer Vulkan anyways on Windows today and as such this is
|
|
# both less variation and closer to what Rerun uses when running on a "real" machine.
|
|
env_vars = setup_lavapipe_for_windows()
|
|
vulkan_info(env_vars)
|
|
elif os.name == "posix" and sys.platform != "darwin" and platform.machine() == "x86_64":
|
|
env_vars = setup_lavapipe_for_linux()
|
|
vulkan_info(env_vars)
|
|
elif os.name == "posix" and sys.platform == "darwin":
|
|
env_vars = setup_lavapipe_for_macos()
|
|
vulkan_info(env_vars)
|
|
else:
|
|
raise ValueError(f"Unsupported OS / architecture: {os.name} / {platform.machine()}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|