fix(instance): mount Hermes/desktop PVCs at /config with legacy layout migration
This commit is contained in:
@@ -442,6 +442,13 @@ data:
|
||||
UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id),
|
||||
INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
023_normalize_runtime_config_mounts.sql: |
|
||||
USE clawreef;
|
||||
UPDATE instances
|
||||
SET mount_path = '/config',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes')
|
||||
AND mount_path IN ('/data', '/home/user/data', '/config/.hermes');
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
UPDATE instances
|
||||
SET mount_path = '/config',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes')
|
||||
AND mount_path IN ('/data', '/home/user/data', '/config/.hermes');
|
||||
@@ -61,7 +61,7 @@ func buildRuntimeConfig(instanceType, osType, osVersion string, registry, tag *s
|
||||
case "hermes":
|
||||
config.Image = defaultSystemImageSettings["hermes"]
|
||||
config.Port = 3001
|
||||
config.MountPath = "/config/.hermes"
|
||||
config.MountPath = "/config"
|
||||
config.Env = defaultWebtopDesktopEnv("Hermes Runtime")
|
||||
case "openclaw":
|
||||
config.MountPath = "/config"
|
||||
@@ -95,7 +95,7 @@ func defaultMountPathForInstanceType(instanceType string) string {
|
||||
case "ubuntu", "webtop", "openclaw":
|
||||
return "/config"
|
||||
case "hermes":
|
||||
return "/config/.hermes"
|
||||
return "/config"
|
||||
default:
|
||||
return "/home/user/data"
|
||||
}
|
||||
|
||||
@@ -28,8 +28,8 @@ func TestBuildRuntimeConfig_HermesUsesWebtopDefaults(t *testing.T) {
|
||||
if config.Port != 3001 {
|
||||
t.Fatalf("expected Hermes port 3001, got %d", config.Port)
|
||||
}
|
||||
if config.MountPath != "/config/.hermes" {
|
||||
t.Fatalf("expected Hermes mount path /config/.hermes, got %q", config.MountPath)
|
||||
if config.MountPath != "/config" {
|
||||
t.Fatalf("expected Hermes mount path /config, got %q", config.MountPath)
|
||||
}
|
||||
if config.Env["SUBFOLDER"] != "/" {
|
||||
t.Fatalf("expected Hermes default SUBFOLDER /, got %q", config.Env["SUBFOLDER"])
|
||||
|
||||
@@ -499,6 +499,7 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
|
||||
EnvFromSecretNames: envFromSecretNames,
|
||||
ExtraPVCMounts: extraPVCMounts,
|
||||
ConfigMapFileMounts: configMapFileMounts,
|
||||
VolumeInitScripts: runtimeVolumeInitScripts(instance.Type, runtimeConfig.MountPath),
|
||||
FSGroup: fsGroup,
|
||||
VolumeOwnershipFixes: volumeOwnershipFixes,
|
||||
SHMSizeGB: shmSizeGB,
|
||||
@@ -642,6 +643,8 @@ func (s *instanceService) Start(instanceID int) error {
|
||||
return fmt.Errorf("failed to build instance agent config: %w", err)
|
||||
}
|
||||
runtimeConfig := buildRuntimeConfig(instance.Type, instance.OSType, instance.OSVersion, instance.ImageRegistry, instance.ImageTag)
|
||||
mountPath := persistentVolumeMountPath(instance)
|
||||
instance.MountPath = mountPath
|
||||
extraEnv, err := buildInstancePodEnv(instance, runtimeConfig.Env, gatewayEnv, agentEnv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve instance environment: %w", err)
|
||||
@@ -673,11 +676,12 @@ func (s *instanceService) Start(instanceID int) error {
|
||||
GPUEnabled: instance.GPUEnabled,
|
||||
GPUCount: instance.GPUCount,
|
||||
Image: runtimeConfig.Image,
|
||||
MountPath: instance.MountPath,
|
||||
MountPath: mountPath,
|
||||
ContainerPort: runtimeConfig.Port,
|
||||
ImagePullPolicy: corev1.PullPolicy(defaultImagePullPolicy()),
|
||||
ExtraEnv: extraEnv,
|
||||
EnvFromSecretNames: []string{bootstrapSecretName},
|
||||
VolumeInitScripts: runtimeVolumeInitScripts(instance.Type, mountPath),
|
||||
SHMSizeGB: shmSizeGB,
|
||||
SecurityMode: s.securityModeForInstance(instance.Type),
|
||||
}
|
||||
@@ -856,12 +860,53 @@ func managedRuntimePersistentDir(instance *models.Instance) string {
|
||||
if strings.EqualFold(instance.Type, "hermes") {
|
||||
return "/config/.hermes"
|
||||
}
|
||||
return persistentVolumeMountPath(instance)
|
||||
}
|
||||
|
||||
func persistentVolumeMountPath(instance *models.Instance) string {
|
||||
if instance == nil {
|
||||
return "/config"
|
||||
}
|
||||
if defaultPath := defaultMountPathForInstanceType(instance.Type); defaultPath == "/config" {
|
||||
return defaultPath
|
||||
}
|
||||
if strings.TrimSpace(instance.MountPath) != "" {
|
||||
return strings.TrimSpace(instance.MountPath)
|
||||
}
|
||||
return defaultMountPathForInstanceType(instance.Type)
|
||||
}
|
||||
|
||||
func runtimeVolumeInitScripts(instanceType, mountPath string) []k8s.VolumeInitScript {
|
||||
if !strings.EqualFold(strings.TrimSpace(instanceType), "hermes") || strings.TrimSpace(mountPath) != "/config" {
|
||||
return nil
|
||||
}
|
||||
return []k8s.VolumeInitScript{
|
||||
{
|
||||
Name: "data",
|
||||
MountPath: "/config",
|
||||
Script: `set -eu
|
||||
base="${CLAWMANAGER_VOLUME_PATH:-/config}"
|
||||
target="$base/.hermes"
|
||||
if [ ! -d "$target" ]; then
|
||||
legacy_found=0
|
||||
for name in hermes-agent skills channels.json session.json bootstrap inventory.json; do
|
||||
if [ -e "$base/$name" ]; then legacy_found=1; fi
|
||||
done
|
||||
mkdir -p "$target"
|
||||
if [ "$legacy_found" = "1" ]; then
|
||||
for entry in "$base"/* "$base"/.[!.]* "$base"/..?*; do
|
||||
[ -e "$entry" ] || continue
|
||||
name="${entry##*/}"
|
||||
case "$name" in .|..|.hermes|Desktop|Downloads|lost+found) continue;; esac
|
||||
mv "$entry" "$target"/
|
||||
done
|
||||
fi
|
||||
fi
|
||||
chown -R 1000:1000 "$target" || true`,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *instanceService) resolveGatewayModelInjection() (*gatewayModelInjection, error) {
|
||||
if s.llmModelRepo == nil {
|
||||
return nil, fmt.Errorf("llm model repository not configured")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"clawreef/internal/models"
|
||||
@@ -131,6 +132,43 @@ func TestBuildAgentEnvInjectsHermesAgentConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistentVolumeMountPathNormalizesManagedDesktopRuntimes(t *testing.T) {
|
||||
for _, instanceType := range []string{"openclaw", "ubuntu", "webtop", "hermes"} {
|
||||
t.Run(instanceType, func(t *testing.T) {
|
||||
got := persistentVolumeMountPath(&models.Instance{
|
||||
Type: instanceType,
|
||||
MountPath: "/data",
|
||||
})
|
||||
if got != "/config" {
|
||||
t.Fatalf("expected %s PVC mount path /config, got %q", instanceType, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRuntimePersistentDirKeepsHermesSubdirectory(t *testing.T) {
|
||||
got := managedRuntimePersistentDir(&models.Instance{
|
||||
Type: "hermes",
|
||||
MountPath: "/config",
|
||||
})
|
||||
if got != "/config/.hermes" {
|
||||
t.Fatalf("expected Hermes persistent dir /config/.hermes, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeVolumeInitScriptsAddsHermesLayoutMigration(t *testing.T) {
|
||||
scripts := runtimeVolumeInitScripts("hermes", "/config")
|
||||
if len(scripts) != 1 {
|
||||
t.Fatalf("expected one Hermes volume init script, got %d", len(scripts))
|
||||
}
|
||||
if scripts[0].Name != "data" || scripts[0].MountPath != "/config" {
|
||||
t.Fatalf("unexpected Hermes volume init script: %#v", scripts[0])
|
||||
}
|
||||
if !strings.Contains(scripts[0].Script, `target="$base/.hermes"`) {
|
||||
t.Fatalf("expected Hermes init script to target /config/.hermes, got %s", scripts[0].Script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveGatewayModelInjectionRequiresActiveModels(t *testing.T) {
|
||||
service := &instanceService{
|
||||
llmModelRepo: &stubLLMModelRepository{},
|
||||
|
||||
@@ -59,6 +59,7 @@ type PodConfig struct {
|
||||
EnvFromSecretNames []string
|
||||
ExtraPVCMounts []PVCMount
|
||||
ConfigMapFileMounts []ConfigMapFileMount
|
||||
VolumeInitScripts []VolumeInitScript
|
||||
FSGroup *int64
|
||||
VolumeOwnershipFixes []VolumeOwnershipFix
|
||||
SHMSizeGB int
|
||||
@@ -88,6 +89,12 @@ type VolumeOwnershipFix struct {
|
||||
GID int64
|
||||
}
|
||||
|
||||
type VolumeInitScript struct {
|
||||
Name string
|
||||
MountPath string
|
||||
Script string
|
||||
}
|
||||
|
||||
// CreatePod creates a new pod for an instance
|
||||
func (s *PodService) CreatePod(ctx context.Context, config PodConfig) (*corev1.Pod, error) {
|
||||
if s.client == nil {
|
||||
@@ -263,6 +270,13 @@ func (s *PodService) CreatePod(ctx context.Context, config PodConfig) (*corev1.P
|
||||
})
|
||||
}
|
||||
|
||||
for index, initScript := range config.VolumeInitScripts {
|
||||
if initScript.Name == "" || initScript.MountPath == "" || initScript.Script == "" {
|
||||
continue
|
||||
}
|
||||
pod.Spec.InitContainers = append(pod.Spec.InitContainers, buildVolumeInitScriptContainer(index, config.Image, pullPolicy, initScript))
|
||||
}
|
||||
|
||||
for index, fix := range config.VolumeOwnershipFixes {
|
||||
if fix.Name == "" || fix.MountPath == "" || fix.UID < 0 || fix.GID < 0 {
|
||||
continue
|
||||
@@ -428,6 +442,33 @@ fi`,
|
||||
}
|
||||
}
|
||||
|
||||
func buildVolumeInitScriptContainer(index int, image string, pullPolicy corev1.PullPolicy, initScript VolumeInitScript) corev1.Container {
|
||||
rootUser := int64(0)
|
||||
return corev1.Container{
|
||||
Name: fmt.Sprintf("init-volume-layout-%d", index+1),
|
||||
Image: image,
|
||||
ImagePullPolicy: pullPolicy,
|
||||
Command: []string{
|
||||
"sh",
|
||||
"-c",
|
||||
initScript.Script,
|
||||
},
|
||||
Env: []corev1.EnvVar{
|
||||
{Name: "CLAWMANAGER_VOLUME_PATH", Value: initScript.MountPath},
|
||||
},
|
||||
SecurityContext: &corev1.SecurityContext{
|
||||
RunAsUser: &rootUser,
|
||||
RunAsGroup: &rootUser,
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{
|
||||
Name: initScript.Name,
|
||||
MountPath: initScript.MountPath,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePodRuntimeType(runtimeType string) string {
|
||||
if runtimeType == "shell" {
|
||||
return "shell"
|
||||
|
||||
@@ -319,7 +319,7 @@ func (s *PVCService) hostPathPVNodeAffinity(ctx context.Context) (*corev1.Volume
|
||||
}
|
||||
candidates := []candidate{}
|
||||
for _, node := range nodes.Items {
|
||||
if node.Spec.Unschedulable || !isStorageNodeReady(node) {
|
||||
if !isHostPathPVNodeCandidate(node) {
|
||||
continue
|
||||
}
|
||||
hostname := strings.TrimSpace(node.Labels["kubernetes.io/hostname"])
|
||||
@@ -367,6 +367,18 @@ func isStorageNodeReady(node corev1.Node) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func isHostPathPVNodeCandidate(node corev1.Node) bool {
|
||||
if node.Spec.Unschedulable || !isStorageNodeReady(node) {
|
||||
return false
|
||||
}
|
||||
for _, taint := range node.Spec.Taints {
|
||||
if taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *PVCService) monitorPVCBinding(ctx context.Context, namespace, pvcName string, userID, instanceID, storageSizeGB int, storageClass string, timeout time.Duration) {
|
||||
if _, err := s.waitForPVCBinding(ctx, namespace, pvcName, userID, instanceID, storageSizeGB, storageClass, timeout); err != nil {
|
||||
fmt.Printf("Async PVC binding monitor failed for %s: %v\n", pvcName, err)
|
||||
@@ -467,6 +479,10 @@ func (s *PVCService) createPVForPVC(ctx context.Context, namespace, pvcName stri
|
||||
}
|
||||
|
||||
storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB))
|
||||
nodeAffinity, err := s.hostPathPVNodeAffinity(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to select node for hostPath PV %s: %w", pvName, err)
|
||||
}
|
||||
|
||||
pv := &corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -487,6 +503,7 @@ func (s *PVCService) createPVForPVC(ctx context.Context, namespace, pvcName stri
|
||||
},
|
||||
PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain,
|
||||
StorageClassName: storageClass,
|
||||
NodeAffinity: nodeAffinity,
|
||||
PersistentVolumeSource: corev1.PersistentVolumeSource{
|
||||
HostPath: &corev1.HostPathVolumeSource{
|
||||
Path: hostPath,
|
||||
|
||||
@@ -105,6 +105,52 @@ func TestHostPathPVNodeAffinitySkipsUnschedulableAndNotReadyNodes(t *testing.T)
|
||||
requireHostnameAffinity(t, affinity, "node-c-host")
|
||||
}
|
||||
|
||||
func TestHostPathPVNodeAffinitySkipsHardTaintedNodes(t *testing.T) {
|
||||
service := &PVCService{
|
||||
client: &Client{
|
||||
Clientset: fake.NewSimpleClientset(
|
||||
&corev1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "k8s-master",
|
||||
Labels: map[string]string{
|
||||
"kubernetes.io/hostname": "k8s-master",
|
||||
},
|
||||
},
|
||||
Spec: corev1.NodeSpec{
|
||||
Taints: []corev1.Taint{
|
||||
{Key: "node-role.kubernetes.io/control-plane", Effect: corev1.TaintEffectNoSchedule},
|
||||
},
|
||||
},
|
||||
Status: corev1.NodeStatus{
|
||||
Conditions: []corev1.NodeCondition{
|
||||
{Type: corev1.NodeReady, Status: corev1.ConditionTrue},
|
||||
},
|
||||
},
|
||||
},
|
||||
&corev1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "k8s-worker1",
|
||||
Labels: map[string]string{
|
||||
"kubernetes.io/hostname": "k8s-worker1",
|
||||
},
|
||||
},
|
||||
Status: corev1.NodeStatus{
|
||||
Conditions: []corev1.NodeCondition{
|
||||
{Type: corev1.NodeReady, Status: corev1.ConditionTrue},
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
affinity, err := service.hostPathPVNodeAffinity(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("hostPathPVNodeAffinity returned error: %v", err)
|
||||
}
|
||||
requireHostnameAffinity(t, affinity, "k8s-worker1")
|
||||
}
|
||||
|
||||
func requireHostnameAffinity(t *testing.T, affinity *corev1.VolumeNodeAffinity, hostname string) {
|
||||
t.Helper()
|
||||
if affinity == nil || affinity.Required == nil || len(affinity.Required.NodeSelectorTerms) != 1 {
|
||||
|
||||
@@ -664,6 +664,13 @@ data:
|
||||
UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id),
|
||||
INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
023_normalize_runtime_config_mounts.sql: |
|
||||
USE clawmanager;
|
||||
UPDATE instances
|
||||
SET mount_path = '/config',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes')
|
||||
AND mount_path IN ('/data', '/home/user/data', '/config/.hermes');
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
|
||||
@@ -668,6 +668,13 @@ data:
|
||||
UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id),
|
||||
INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
023_normalize_runtime_config_mounts.sql: |
|
||||
USE clawmanager;
|
||||
UPDATE instances
|
||||
SET mount_path = '/config',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes')
|
||||
AND mount_path IN ('/data', '/home/user/data', '/config/.hermes');
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
|
||||
@@ -12,11 +12,12 @@ A Hermes image must satisfy two layers of requirements:
|
||||
Current Hermes runtime defaults in ClawManager:
|
||||
|
||||
- Port: `3001`
|
||||
- PVC mount path: `/config`
|
||||
- Persistent directory: `/config/.hermes`
|
||||
- Default title: `Hermes Runtime`
|
||||
- Proxy path: ClawManager rewrites `SUBFOLDER` to `/api/v1/instances/{instance_id}/proxy/` when the instance is created.
|
||||
|
||||
Do not change the port or persistent directory in the image. If the image uses a different port or mount path, instance proxying, PVC mounting, and user data persistence will no longer match ClawManager expectations.
|
||||
Do not change the port, PVC mount path, or persistent directory in the image. ClawManager mounts the instance PVC at `/config` so Webtop desktop files such as `/config/Desktop` and Hermes runtime files under `/config/.hermes` persist together.
|
||||
|
||||
## Image Build Requirements
|
||||
|
||||
@@ -799,7 +800,7 @@ Never commit real tokens, channel secrets, Gateway API keys, or downloaded sessi
|
||||
Before delivering a Hermes image, verify:
|
||||
|
||||
- The Webtop desktop is reachable through the ClawManager instance proxy.
|
||||
- `/config/.hermes` is mounted and persists across restarts.
|
||||
- `/config` is mounted and both `/config/Desktop` and `/config/.hermes` persist across restarts.
|
||||
- The agent registers and starts heartbeat within 30 seconds.
|
||||
- The instance detail page shows agent online, runtime running, and an updated last report time.
|
||||
- CPU, memory, disk, and network metrics are visible and refresh continuously.
|
||||
@@ -819,7 +820,7 @@ ClawManager must keep the following capabilities for Hermes to work end to end:
|
||||
- Allow `hermes` instances to register with the Agent Control Plane.
|
||||
- Inject `CLAWMANAGER_LLM_*` and OpenAI-compatible variables so Hermes can access models through ClawManager AI Gateway.
|
||||
- Inject Hermes and generic runtime bootstrap variables for channels, skills, and related resources.
|
||||
- Mount persistent storage at `/config/.hermes`.
|
||||
- Mount persistent storage at `/config`, while keeping Hermes runtime state under `/config/.hermes`.
|
||||
- Support `.hermes` import and export.
|
||||
- Keep compatibility fields such as `openclaw_status`, `openclaw_pid`, and `openclaw_version` until generic runtime fields are introduced.
|
||||
- Add Hermes-specific runtime control commands, or generic `start_runtime`, `stop_runtime`, and `restart_runtime`, if runtime process control becomes required.
|
||||
|
||||
Reference in New Issue
Block a user