mirrors/deepspeedai--deepspeed
1
0
Fork 0
Code Issues Pull Requests Actions 13 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:18:33 +08:00
commit 4ececc111a
2017 changed files with 331736 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deepspeed/runtime/superoffload/__init__.py
+4
View File
@@ -0,0 +1,4 @@
# Copyright (c) DeepSpeed Team.
# SPDX-License-Identifier: Apache-2.0
# DeepSpeed Team
deepspeed/runtime/superoffload/superoffload_stage3.py
+375
View File
@@ -0,0 +1,375 @@
# Copyright (c) DeepSpeed Team.
# SPDX-License-Identifier: Apache-2.0
# DeepSpeed Team
import time
import torch
from typing import List
from deepspeed.runtime.superoffload.superoffload_utils import SuperOffloadCPUOptimizer, TaskKeys, ResultKeys, EventTypes
from deepspeed.runtime.zero.partition_parameters import Parameter, Tensor
from deepspeed.runtime.zero.stage3 import DeepSpeedZeroOptimizer_Stage3
from deepspeed.utils.nvtx import instrument_w_nvtx
from deepspeed.utils import logger
from deepspeed.accelerator import get_accelerator
OPTIMIZER_STEP_TIMER = 'optimizer_step'
def _validate_superoffload_accelerator():
"""Validate that the current accelerator is compatible with SuperOffload."""
accelerator = get_accelerator()
assert accelerator.device_name() == 'cuda', (
f"SuperOffload only supports NVIDIA CUDA GPUs, but found accelerator '{accelerator.device_name()}'.")
class SuperOffloadOptimizer_Stage3(DeepSpeedZeroOptimizer_Stage3):
def __init__(
self,
module,
init_optimizer,
param_names,
timers,
ds_config,
**kwargs,
):
_validate_superoffload_accelerator()
self.sub_group_to_param_num = {}
self.sub_group_grad_partition_counts = {}
self.async_cpuadam_num = 0
self.max_grad_numel = 0
super().__init__(module, init_optimizer, param_names, timers, ds_config, **kwargs)
optimizer_configs = []
for pg in self.optimizer.param_groups:
optimizer_configs.append({
"lr": pg["lr"],
"betas": pg["betas"],
"eps": pg["eps"],
"weight_decay": pg["weight_decay"],
"amsgrad": pg["amsgrad"],
})
cpuadam_cores_perc = kwargs.get("cpuadam_cores_perc", 0.8)
self.superoffload_cpu_optimizer = SuperOffloadCPUOptimizer(optimizer_config=optimizer_configs,
cpuadam_cores_perc=cpuadam_cores_perc,
max_grad_numel=self.max_grad_numel)
def _create_fp16_sub_groups(self, params_group):
params_group_numel = sum([param.partition_numel() for param in params_group])
sub_group_size = self.sub_group_size
if sub_group_size is None or sub_group_size >= params_group_numel:
global_idx = len(self.sub_group_to_param_num)
self.sub_group_to_param_num[global_idx] = len(params_group)
self.max_grad_numel = max(self.max_grad_numel, params_group_numel)
return [params_group]
sub_groups = []
sub_group = []
local_sub_group_size = 0
for param in params_group:
sub_group.append(param)
local_sub_group_size += param.partition_numel()
if local_sub_group_size >= sub_group_size or id(param) == id(params_group[-1]):
self.max_grad_numel = max(self.max_grad_numel, local_sub_group_size)
sub_groups.append(sub_group)
global_idx = len(self.sub_group_to_param_num)
self.sub_group_to_param_num[global_idx] = len(sub_group)
sub_group = []
local_sub_group_size = 0
return sub_groups
def _optimizer_step(self, sub_group_id):
param_group_id = self.sub_group_to_group_id[sub_group_id]
fp32_param = self.fp32_partitioned_groups_flat[sub_group_id]
def step_with_gradscaler(optimizer):
if self.torch_autocast_gradscaler:
self.torch_autocast_gradscaler.step(optimizer)
self.torch_autocast_gradscaler.update()
else:
optimizer.step()
cur_device = self.subgroup_to_device[sub_group_id]
if cur_device != 'cpu':
self.backup_optimizer.param_groups[param_group_id]['params'] = [fp32_param]
step_with_gradscaler(self.backup_optimizer)
self.backup_optimizer.param_groups[param_group_id]['params'] = []
@instrument_w_nvtx
def independent_gradient_partition_epilogue(self):
super().independent_gradient_partition_epilogue()
self.sub_group_grad_partition_counts.clear()
@instrument_w_nvtx
def _reassign_or_swap_out_partitioned_parameters(self, sub_group_id):
if self.subgroup_to_device[sub_group_id] == 'cpu':
self.fp16_partitioned_groups_flat[sub_group_id].data.copy_(
self.fp32_partitioned_groups_flat[sub_group_id].data)
self._unflatten_partitioned_parameters(sub_group_id)
return
if self.fp16_partitioned_groups_flat[sub_group_id] is not None:
self.fp16_partitioned_groups_flat[sub_group_id].data.copy_(
self.fp32_partitioned_groups_flat[sub_group_id].data)
self._unflatten_partitioned_parameters(sub_group_id)
else:
self._partitioned_params_swap_out(sub_group_id)
@instrument_w_nvtx
def _reassign_or_swap_out_partitioned_parameters_async(self, sub_group_id, updated_param):
"""Asynchronously update partitioned parameters with optimized values."""
self.fp32_partitioned_groups_flat[sub_group_id].data.copy_(updated_param, non_blocking=True)
@instrument_w_nvtx
def partition_grads(self, params_to_release: List[Parameter], grad_partitions: List[Tensor]) -> None:
completed_sub_groups = []
for param, grad_partition in zip(params_to_release, grad_partitions):
i, dest_offset, _ = self.grad_position[self.get_param_id(param)]
# Accumulate gradient into the grad_buffer, mirroring base class logic
grad_buffer = self._DeepSpeedZeroOptimizer_Stage3__param_id_to_grad_partition[param.ds_id].narrow(
0, 0, grad_partition.numel())
if self.micro_step_id == 0:
grad_buffer.copy_(grad_partition, non_blocking=True)
grad_buffer = grad_buffer.to(grad_partition.device, non_blocking=True)
elif get_accelerator().on_accelerator(grad_buffer):
grad_buffer.add_(grad_partition.to(self.gradient_accumulation_dtype).view(grad_buffer.shape))
else:
cuda_grad_buffer = grad_buffer.to(grad_partition.device, non_blocking=True)
cuda_grad_buffer.add_(grad_partition.to(self.gradient_accumulation_dtype).view(cuda_grad_buffer.shape))
grad_buffer.copy_(cuda_grad_buffer, non_blocking=True)
grad_buffer = cuda_grad_buffer
if self.is_gradient_accumulation_boundary:
self.norm_for_param_grads[self.get_param_id(param)] = self._constant_buffered_norm2(grad_buffer)
fp32_grad_tensor = self.fp32_partitioned_groups_flat[i].grad.narrow(
0, dest_offset, grad_buffer.numel())
fp32_grad_tensor.copy_(grad_buffer.to(dtype=self.master_weights_and_grads_dtype), non_blocking=True)
self.sub_group_grad_partition_counts[i] = self.sub_group_grad_partition_counts.get(i, 0) + 1
if self.sub_group_grad_partition_counts[i] == self.sub_group_to_param_num[i]:
completed_sub_groups.append(i)
if self.is_gradient_accumulation_boundary and completed_sub_groups:
get_accelerator().current_stream().synchronize()
for i in completed_sub_groups:
if self.subgroup_to_device[i] == 'cpu' and not self.clip_grad:
param_group_id = self.sub_group_to_group_id[i]
fp32_param = self.fp32_partitioned_groups_flat[i]
current_lr = self.optimizer.param_groups[param_group_id]['lr']
self.superoffload_cpu_optimizer.async_step(param_group_id,
i,
fp32_param.data,
fp32_param.grad.data,
lr=current_lr)
self.async_cpuadam_num += 1
result = self.superoffload_cpu_optimizer.get_result()
if result is not None:
self._reassign_or_swap_out_partitioned_parameters_async(result[TaskKeys.SUB_GROUP_ID],
result[ResultKeys.UPDATED_PARAM])
self.async_cpuadam_num -= 1
for param in params_to_release:
if not get_accelerator().is_synchronized_device():
if param.grad is not None:
param.grad.record_stream(get_accelerator().current_stream())
param.grad = None
@instrument_w_nvtx
def step(self, closure=None):
"""
Not supporting closure.
"""
self._wait_for_async_operations()
self._pre_step()
self._partition_all_parameters()
if self._overflow_check_and_loss_scale_update():
if not self.clip_grad:
self._handle_overflow_rollback()
return
norm_groups = self._get_norm_groups()
scaled_global_grad_norm = torch.linalg.vector_norm(torch.stack(norm_groups))
self._global_grad_norm = scaled_global_grad_norm / self.loss_scale
timer_names = set()
timer_names.add(OPTIMIZER_STEP_TIMER)
self.timers(OPTIMIZER_STEP_TIMER).start()
if self.clip_grad:
self._step_with_clipping(scaled_global_grad_norm, timer_names)
else:
self._step_without_clipping(scaled_global_grad_norm, timer_names)
self.timers(OPTIMIZER_STEP_TIMER).stop()
self._post_step(timer_names)
def _step_without_clipping(self, scaled_global_grad_norm, timer_names):
"""Fast path: async CPU steps already completed during backward."""
for sub_group_id, group in enumerate(self.fp16_groups):
self._prepare_sub_group(sub_group_id, timer_names)
self.unscale_and_clip_grads(sub_group_id, scaled_global_grad_norm)
self._optimizer_step(sub_group_id)
self._reassign_or_swap_out_partitioned_parameters(sub_group_id)
self._release_sub_group(sub_group_id, timer_names)
def _step_with_clipping(self, scaled_global_grad_norm, timer_names):
"""Clipping path: no async steps were done during backward,
so we unscale+clip first, then step all sub-groups."""
for sub_group_id, group in enumerate(self.fp16_groups):
self._prepare_sub_group(sub_group_id, timer_names)
self.unscale_and_clip_grads(sub_group_id, scaled_global_grad_norm)
if self.subgroup_to_device[sub_group_id] == 'cpu':
param_group_id = self.sub_group_to_group_id[sub_group_id]
fp32_param = self.fp32_partitioned_groups_flat[sub_group_id]
current_lr = self.optimizer.param_groups[param_group_id]['lr']
self._sync_cpu_optimizer_step(param_group_id,
sub_group_id,
fp32_param.data,
fp32_param.grad.data,
lr=current_lr)
else:
self._optimizer_step(sub_group_id)
self._reassign_or_swap_out_partitioned_parameters(sub_group_id)
self._release_sub_group(sub_group_id, timer_names)
def _wait_for_async_operations(self, timeout_seconds=60):
"""Wait for all pending asynchronous CPU optimizer operations to complete with timeout error.
Args:
timeout_seconds (int): Maximum time to wait before throwing an error. Default is 60 seconds.
"""
if self.async_cpuadam_num > 0:
logger.info(f"[INFO] {self.async_cpuadam_num} asynchronous CPU optimizer operations pending...")
if self.async_cpuadam_num == 0:
return
start_time = time.time()
initial_pending_ops = self.async_cpuadam_num
while self.async_cpuadam_num > 0:
result = self.superoffload_cpu_optimizer.get_result()
if result is None:
current_time = time.time()
elapsed_time = current_time - start_time
# Throw error if we've been waiting longer than the timeout
if elapsed_time >= timeout_seconds:
raise RuntimeError(
f"SuperOffload CPU optimizer timeout after {elapsed_time:.1f} seconds. "
f"Still waiting for {self.async_cpuadam_num}/{initial_pending_ops} async operations to complete. "
f"This indicates a deadlock or critical performance issue in the CPU optimizer.")
time.sleep(0.001) # 1ms sleep
continue
self._reassign_or_swap_out_partitioned_parameters_async(result[TaskKeys.SUB_GROUP_ID],
result[ResultKeys.UPDATED_PARAM])
self.async_cpuadam_num -= 1
def _wait_for_single_async_result(self, event_type: str, timeout_seconds=60):
"""Wait for a single asynchronous CPU-Adam optimizer operation with timeout.
Args:
event_type (str): Type of operation expected ('adam_step' or 'rollback').
timeout_seconds (int): Maximum time to wait before throwing an error. Default is 60 seconds.
"""
start_time = time.time()
while True:
result = self.superoffload_cpu_optimizer.get_result(expected_event_type=event_type)
if result is not None:
self._reassign_or_swap_out_partitioned_parameters_async(result[TaskKeys.SUB_GROUP_ID],
result[ResultKeys.UPDATED_PARAM])
break
current_time = time.time()
elapsed_time = current_time - start_time
# Throw error if we've been waiting longer than the timeout
if elapsed_time >= timeout_seconds:
raise RuntimeError(f"SuperOffload CPU optimizer timeout after {elapsed_time:.1f} seconds. "
f"This indicates a deadlock or critical performance issue in the CPU optimizer.")
time.sleep(0.001) # 1ms sleep
def _sync_cpu_optimizer_step(self,
param_group_id: int,
sub_group_id: int,
fp32_param_data,
fp32_grad_data,
rollback: bool = False,
lr: float = None,
timeout_seconds: int = 60):
event_type = EventTypes.ROLLBACK if rollback else EventTypes.ADAM_STEP
self.superoffload_cpu_optimizer.async_step(param_group_id,
sub_group_id,
fp32_param_data,
fp32_grad_data,
rollback=rollback,
lr=lr)
# Wait for completion
self._wait_for_single_async_result(event_type, timeout_seconds)
def _handle_overflow_rollback(self):
"""Handle gradient overflow by rolling back CPU optimizer states."""
for sub_group_id, _ in enumerate(self.fp16_groups):
if self.subgroup_to_device[sub_group_id] == 'cpu':
param_group_id = self.sub_group_to_group_id[sub_group_id]
fp32_param = self.fp32_partitioned_groups_flat[sub_group_id]
# Trigger rollback
self._sync_cpu_optimizer_step(param_group_id,
sub_group_id,
fp32_param.data,
fp32_param.grad.data,
rollback=True)
def _handle_gradient_clipping(self, scaled_global_grad_norm):
"""Handle gradient clipping with CPU optimizer rollback and re-optimization."""
for sub_group_id, _ in enumerate(self.fp16_groups):
if self.subgroup_to_device[sub_group_id] == 'cpu':
param_group_id = self.sub_group_to_group_id[sub_group_id]
fp32_param = self.fp32_partitioned_groups_flat[sub_group_id]
# Rollback CPU optimizer states
self._sync_cpu_optimizer_step(param_group_id,
sub_group_id,
fp32_param.data,
fp32_param.grad.data,
rollback=True)
# Clip gradients and re-optimize
self.unscale_and_clip_grads(sub_group_id, scaled_global_grad_norm)
current_lr = self.optimizer.param_groups[param_group_id]['lr']
self._sync_cpu_optimizer_step(param_group_id,
sub_group_id,
fp32_param.data,
fp32_param.grad.data,
rollback=False,
lr=current_lr)
@instrument_w_nvtx
def check_clip_grads(self, total_norm):
"""Check if gradients need to be clipped based on the global norm."""
unscaled_norm = total_norm / self.loss_scale
return self.clip_grad and unscaled_norm > self.clip_grad
deepspeed/runtime/superoffload/superoffload_utils.py
+297
View File
@@ -0,0 +1,297 @@
# Copyright (c) DeepSpeed Team.
# SPDX-License-Identifier: Apache-2.0
# DeepSpeed Team
"""
SuperOffload utilities for 1) running CPU optimizers in separate processes.
"""
from typing import Dict, Optional, Any
import torch
import torch.multiprocessing as mp
import psutil
from deepspeed.ops.adam import DeepSpeedCPUAdam
from deepspeed.utils import logger
class TaskKeys:
PARAM_DATA = "param_data"
PARAM_GRAD = "param_grad"
PARAM_GROUP_ID = "param_group_id"
SUB_GROUP_ID = "sub_group_id"
ROLLBACK = "rollback"
LR = "lr"
class ResultKeys:
UPDATED_PARAM = "updated_param"
EVENT_TYPE = "event_type"
class EventTypes:
ADAM_STEP = "adam_step"
ROLLBACK = "rollback"
def superoffload_optimizer_worker(param_queue: mp.SimpleQueue, result_queue: mp.SimpleQueue,
optimizer_config: Dict[str, Any], max_grad_numel: int) -> None:
"""
This function runs in a separate process and continuously processes optimization
tasks from the parameter queue. It creates a DeepSpeedCPUAdam optimizer and
applies optimization steps to parameters received from the main process.
Args:
param_queue: Queue for receiving optimization tasks
result_queue: Queue for sending back optimization results
optimizer_config: Configuration dictionary for the optimizer containing
lr, betas, eps, weight_decay, and amsgrad parameters
max_grad_numel: Maximum number of elements expected in gradient tensors
"""
cpu_tensor = torch.randn(1, device="cpu")
cpu_param = torch.nn.Parameter(cpu_tensor)
try:
if isinstance(optimizer_config, list):
pg_configs = optimizer_config
else:
pg_configs = [optimizer_config]
first_cfg = pg_configs[0]
optimizer = DeepSpeedCPUAdam([cpu_param],
lr=first_cfg["lr"],
betas=first_cfg["betas"],
eps=first_cfg["eps"],
weight_decay=first_cfg["weight_decay"],
amsgrad=first_cfg["amsgrad"])
for cfg in pg_configs[1:]:
dummy = torch.nn.Parameter(torch.randn(1, device="cpu"))
optimizer.add_param_group({
"params": [dummy],
"lr": cfg["lr"],
"betas": cfg["betas"],
"eps": cfg["eps"],
"weight_decay": cfg["weight_decay"],
"amsgrad": cfg["amsgrad"],
})
except KeyError as e:
error_msg = f"Missing required optimizer config key: {e}"
logger.error(error_msg)
result_queue.put({"error": error_msg})
return
# Pre-allocate reusable pinned memory buffer for gradients
pinned_grad_buffer = torch.empty(max_grad_numel, dtype=torch.float32, device='cpu', pin_memory=True)
while True:
try:
task = param_queue.get()
if task is None:
logger.debug("Received termination signal, shutting down worker")
break
param_data = task[TaskKeys.PARAM_DATA]
param_grad = task[TaskKeys.PARAM_GRAD]
param_group_id = task[TaskKeys.PARAM_GROUP_ID]
sub_group_id = task[TaskKeys.SUB_GROUP_ID]
rollback = task.get(TaskKeys.ROLLBACK, False)
task_lr = task.get(TaskKeys.LR, None)
logger.debug(f"Processing param_group_id: {param_group_id}, sub_group_id: {sub_group_id}")
del task[TaskKeys.PARAM_DATA]
del task[TaskKeys.PARAM_GRAD]
task.clear()
if task_lr is not None:
optimizer.param_groups[param_group_id]['lr'] = task_lr
grad_numel = param_grad.numel()
if grad_numel > max_grad_numel:
error_msg = (
f"Gradient size {grad_numel} exceeds pre-allocated buffer size {max_grad_numel}. "
f"This indicates insufficient buffer allocation. Please increase max_grad_numel parameter.")
result_queue.put({"error": error_msg})
break
param_grad_cpu = pinned_grad_buffer[:grad_numel].view_as(param_grad)
param_grad_cpu.copy_(param_grad, non_blocking=False)
fp32_param = torch.nn.Parameter(param_data)
fp32_param.grad = param_grad_cpu
optimizer.param_groups[param_group_id]['params'] = [fp32_param]
if rollback:
logger.debug(f"Rolling back optimizer state for sub_group_id: {sub_group_id}")
optimizer.rollback_subgroup(sub_group_id)
else:
optimizer.step_subgroup(sub_group_id)
# Send result back to main process
event_type = EventTypes.ROLLBACK if rollback else EventTypes.ADAM_STEP
result_queue.put({
TaskKeys.PARAM_GROUP_ID: param_group_id,
TaskKeys.SUB_GROUP_ID: sub_group_id,
ResultKeys.UPDATED_PARAM: fp32_param.data,
ResultKeys.EVENT_TYPE: event_type,
})
# Clean up references to free memory
optimizer.param_groups[param_group_id]['params'] = []
del param_grad_cpu, fp32_param.grad, fp32_param, param_grad, param_data
except KeyError as e:
error_msg = f"Missing required task key: {e}"
logger.error(error_msg)
result_queue.put({"error": error_msg})
break
except Exception as e:
error_msg = f"Unexpected error in worker process: {e}"
logger.error(error_msg)
result_queue.put({"error": error_msg})
break
# Clean up pinned memory buffer
if 'pinned_grad_buffer' in locals():
del pinned_grad_buffer
logger.debug("Cleaned up pinned memory buffer")
logger.debug("Worker process terminated")
class SuperOffloadCPUOptimizer:
def __init__(self,
optimizer_config: Dict[str, Any],
cpuadam_cores_perc: float = 0.8,
max_grad_numel: int = 1000000) -> None:
if not 0 < cpuadam_cores_perc <= 1:
raise ValueError("cpuadam_cores_perc must be between 0 and 1")
self.max_grad_numel = max_grad_numel
self.mp_context = mp.get_context('spawn')
self.param_queue = self.mp_context.SimpleQueue()
self.result_queue = self.mp_context.SimpleQueue()
self.cpuadam_process = self.mp_context.Process(
target=superoffload_optimizer_worker,
args=(self.param_queue, self.result_queue, optimizer_config, max_grad_numel),
daemon=True,
)
self.cpuadam_process.start()
# Set CPU affinity for better performance isolation
self._set_cpu_affinity(cpuadam_cores_perc)
def _set_cpu_affinity(self, cpuadam_cores_perc: float) -> None:
"""
Set CPU affinity for the main (Pytorch) process and worker (CPU Adam) process.
Args:
cpuadam_cores_perc: Percentage of cores to allocate to the worker (CPU Adam) process
"""
try:
current_process = psutil.Process()
all_cores = current_process.cpu_affinity()
num_cores = len(all_cores)
split_idx = int((1 - cpuadam_cores_perc) * num_cores)
pt_cores = all_cores[:split_idx]
cpuadam_cores = all_cores[split_idx:]
# Set affinity for main process (PyTorch)
current_process.cpu_affinity(pt_cores)
# Set affinity for optimizer process (CPU Adam)
optimizer_process = psutil.Process(self.cpuadam_process.pid)
optimizer_process.cpu_affinity(cpuadam_cores)
logger.debug(f"Set CPU affinity - PyTorch cores: {pt_cores}, "
f"Optimizer cores: {cpuadam_cores}")
except (psutil.AccessDenied, psutil.NoSuchProcess, AttributeError) as e:
logger.debug(f"Could not set CPU affinities for superoffload optimizer process: {e}")
except Exception as e:
logger.warning(f"Unexpected error setting CPU affinity: {e}")
def async_step(self,
param_group_id: int,
sub_group_id: int,
fp32_param: torch.Tensor,
fp32_grad: torch.Tensor,
rollback: bool = False,
lr: float = None) -> None:
"""
Queue parameter for optimization in the worker process.
"""
if not self.cpuadam_process.is_alive():
raise RuntimeError("Worker process is not alive")
task = {
TaskKeys.PARAM_DATA: fp32_param,
TaskKeys.PARAM_GRAD: fp32_grad,
TaskKeys.PARAM_GROUP_ID: param_group_id,
TaskKeys.SUB_GROUP_ID: sub_group_id,
TaskKeys.ROLLBACK: rollback,
}
if lr is not None:
task[TaskKeys.LR] = lr
self.param_queue.put(task)
def get_result(self, expected_event_type: str = None) -> Optional[Dict[str, Any]]:
"""
Get result from worker process with optional event type validation.
Args:
expected_event_type (str, optional): Expected event type ('adam_step' or 'rollback').
If provided, validates that the result matches.
"""
if self.result_queue.empty():
return None
result = self.result_queue.get()
if "error" in result:
raise RuntimeError(f"Error in worker process: {result['error']}")
# Validate event type if expected_event_type is provided
if expected_event_type is not None:
result_event_type = result.get(ResultKeys.EVENT_TYPE)
if result_event_type != expected_event_type:
raise RuntimeError(f"Event type mismatch: expected '{expected_event_type}', got '{result_event_type}'")
return result
def close(self) -> None:
"""
Shutdown the worker process gracefully.
Sends termination signal to worker and waits for clean shutdown.
If the process doesn't terminate within the timeout, it will be forcefully killed.
"""
if not self.cpuadam_process.is_alive():
logger.debug("Worker process already terminated")
return
# Send termination signal
self.param_queue.put(None)
# Wait for graceful shutdown
self.cpuadam_process.join(timeout=5)
if self.cpuadam_process.is_alive():
logger.warning("Optimizer process did not terminate cleanly within timeout, "
"forcefully terminating")
self.cpuadam_process.terminate()
self.cpuadam_process.join(timeout=2)
# Last resort: kill the process
if self.cpuadam_process.is_alive():
logger.error("Failed to terminate optimizer process, killing it")
self.cpuadam_process.kill()
self.cpuadam_process.join()
logger.debug("SuperOffload CPU optimizer closed successfully")