diff --git a/backend/deployments/k8s/clawreef-incluster.yaml b/backend/deployments/k8s/clawreef-incluster.yaml index 4614137..f306284 100644 --- a/backend/deployments/k8s/clawreef-incluster.yaml +++ b/backend/deployments/k8s/clawreef-incluster.yaml @@ -442,6 +442,13 @@ data: UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 023_normalize_runtime_config_mounts.sql: | + USE clawreef; + UPDATE instances + SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP + WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); --- apiVersion: v1 kind: PersistentVolumeClaim diff --git a/backend/internal/db/migrations/023_normalize_runtime_config_mounts.sql b/backend/internal/db/migrations/023_normalize_runtime_config_mounts.sql new file mode 100644 index 0000000..bf5ce0b --- /dev/null +++ b/backend/internal/db/migrations/023_normalize_runtime_config_mounts.sql @@ -0,0 +1,5 @@ +UPDATE instances +SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP +WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); diff --git a/backend/internal/services/instance_runtime.go b/backend/internal/services/instance_runtime.go index 81ba1b8..470ebc1 100644 --- a/backend/internal/services/instance_runtime.go +++ b/backend/internal/services/instance_runtime.go @@ -61,7 +61,7 @@ func buildRuntimeConfig(instanceType, osType, osVersion string, registry, tag *s case "hermes": config.Image = defaultSystemImageSettings["hermes"] config.Port = 3001 - config.MountPath = "/config/.hermes" + config.MountPath = "/config" config.Env = defaultWebtopDesktopEnv("Hermes Runtime") case "openclaw": config.MountPath = "/config" @@ -95,7 +95,7 @@ func defaultMountPathForInstanceType(instanceType string) string { case "ubuntu", "webtop", "openclaw": return "/config" case "hermes": - return "/config/.hermes" + return "/config" default: return "/home/user/data" } diff --git a/backend/internal/services/instance_runtime_test.go b/backend/internal/services/instance_runtime_test.go index b7a0caa..888d84b 100644 --- a/backend/internal/services/instance_runtime_test.go +++ b/backend/internal/services/instance_runtime_test.go @@ -28,8 +28,8 @@ func TestBuildRuntimeConfig_HermesUsesWebtopDefaults(t *testing.T) { if config.Port != 3001 { t.Fatalf("expected Hermes port 3001, got %d", config.Port) } - if config.MountPath != "/config/.hermes" { - t.Fatalf("expected Hermes mount path /config/.hermes, got %q", config.MountPath) + if config.MountPath != "/config" { + t.Fatalf("expected Hermes mount path /config, got %q", config.MountPath) } if config.Env["SUBFOLDER"] != "/" { t.Fatalf("expected Hermes default SUBFOLDER /, got %q", config.Env["SUBFOLDER"]) diff --git a/backend/internal/services/instance_service.go b/backend/internal/services/instance_service.go index e54dd94..6439f22 100644 --- a/backend/internal/services/instance_service.go +++ b/backend/internal/services/instance_service.go @@ -499,6 +499,7 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models EnvFromSecretNames: envFromSecretNames, ExtraPVCMounts: extraPVCMounts, ConfigMapFileMounts: configMapFileMounts, + VolumeInitScripts: runtimeVolumeInitScripts(instance.Type, runtimeConfig.MountPath), FSGroup: fsGroup, VolumeOwnershipFixes: volumeOwnershipFixes, SHMSizeGB: shmSizeGB, @@ -642,6 +643,8 @@ func (s *instanceService) Start(instanceID int) error { return fmt.Errorf("failed to build instance agent config: %w", err) } runtimeConfig := buildRuntimeConfig(instance.Type, instance.OSType, instance.OSVersion, instance.ImageRegistry, instance.ImageTag) + mountPath := persistentVolumeMountPath(instance) + instance.MountPath = mountPath extraEnv, err := buildInstancePodEnv(instance, runtimeConfig.Env, gatewayEnv, agentEnv) if err != nil { return fmt.Errorf("failed to resolve instance environment: %w", err) @@ -673,11 +676,12 @@ func (s *instanceService) Start(instanceID int) error { GPUEnabled: instance.GPUEnabled, GPUCount: instance.GPUCount, Image: runtimeConfig.Image, - MountPath: instance.MountPath, + MountPath: mountPath, ContainerPort: runtimeConfig.Port, ImagePullPolicy: corev1.PullPolicy(defaultImagePullPolicy()), ExtraEnv: extraEnv, EnvFromSecretNames: []string{bootstrapSecretName}, + VolumeInitScripts: runtimeVolumeInitScripts(instance.Type, mountPath), SHMSizeGB: shmSizeGB, SecurityMode: s.securityModeForInstance(instance.Type), } @@ -856,12 +860,53 @@ func managedRuntimePersistentDir(instance *models.Instance) string { if strings.EqualFold(instance.Type, "hermes") { return "/config/.hermes" } + return persistentVolumeMountPath(instance) +} + +func persistentVolumeMountPath(instance *models.Instance) string { + if instance == nil { + return "/config" + } + if defaultPath := defaultMountPathForInstanceType(instance.Type); defaultPath == "/config" { + return defaultPath + } if strings.TrimSpace(instance.MountPath) != "" { return strings.TrimSpace(instance.MountPath) } return defaultMountPathForInstanceType(instance.Type) } +func runtimeVolumeInitScripts(instanceType, mountPath string) []k8s.VolumeInitScript { + if !strings.EqualFold(strings.TrimSpace(instanceType), "hermes") || strings.TrimSpace(mountPath) != "/config" { + return nil + } + return []k8s.VolumeInitScript{ + { + Name: "data", + MountPath: "/config", + Script: `set -eu +base="${CLAWMANAGER_VOLUME_PATH:-/config}" +target="$base/.hermes" +if [ ! -d "$target" ]; then + legacy_found=0 + for name in hermes-agent skills channels.json session.json bootstrap inventory.json; do + if [ -e "$base/$name" ]; then legacy_found=1; fi + done + mkdir -p "$target" + if [ "$legacy_found" = "1" ]; then + for entry in "$base"/* "$base"/.[!.]* "$base"/..?*; do + [ -e "$entry" ] || continue + name="${entry##*/}" + case "$name" in .|..|.hermes|Desktop|Downloads|lost+found) continue;; esac + mv "$entry" "$target"/ + done + fi +fi +chown -R 1000:1000 "$target" || true`, + }, + } +} + func (s *instanceService) resolveGatewayModelInjection() (*gatewayModelInjection, error) { if s.llmModelRepo == nil { return nil, fmt.Errorf("llm model repository not configured") diff --git a/backend/internal/services/instance_service_test.go b/backend/internal/services/instance_service_test.go index ea0150b..a91656c 100644 --- a/backend/internal/services/instance_service_test.go +++ b/backend/internal/services/instance_service_test.go @@ -1,6 +1,7 @@ package services import ( + "strings" "testing" "clawreef/internal/models" @@ -131,6 +132,43 @@ func TestBuildAgentEnvInjectsHermesAgentConfig(t *testing.T) { } } +func TestPersistentVolumeMountPathNormalizesManagedDesktopRuntimes(t *testing.T) { + for _, instanceType := range []string{"openclaw", "ubuntu", "webtop", "hermes"} { + t.Run(instanceType, func(t *testing.T) { + got := persistentVolumeMountPath(&models.Instance{ + Type: instanceType, + MountPath: "/data", + }) + if got != "/config" { + t.Fatalf("expected %s PVC mount path /config, got %q", instanceType, got) + } + }) + } +} + +func TestManagedRuntimePersistentDirKeepsHermesSubdirectory(t *testing.T) { + got := managedRuntimePersistentDir(&models.Instance{ + Type: "hermes", + MountPath: "/config", + }) + if got != "/config/.hermes" { + t.Fatalf("expected Hermes persistent dir /config/.hermes, got %q", got) + } +} + +func TestRuntimeVolumeInitScriptsAddsHermesLayoutMigration(t *testing.T) { + scripts := runtimeVolumeInitScripts("hermes", "/config") + if len(scripts) != 1 { + t.Fatalf("expected one Hermes volume init script, got %d", len(scripts)) + } + if scripts[0].Name != "data" || scripts[0].MountPath != "/config" { + t.Fatalf("unexpected Hermes volume init script: %#v", scripts[0]) + } + if !strings.Contains(scripts[0].Script, `target="$base/.hermes"`) { + t.Fatalf("expected Hermes init script to target /config/.hermes, got %s", scripts[0].Script) + } +} + func TestResolveGatewayModelInjectionRequiresActiveModels(t *testing.T) { service := &instanceService{ llmModelRepo: &stubLLMModelRepository{}, diff --git a/backend/internal/services/k8s/pod_service.go b/backend/internal/services/k8s/pod_service.go index eaa6953..c8ff732 100644 --- a/backend/internal/services/k8s/pod_service.go +++ b/backend/internal/services/k8s/pod_service.go @@ -59,6 +59,7 @@ type PodConfig struct { EnvFromSecretNames []string ExtraPVCMounts []PVCMount ConfigMapFileMounts []ConfigMapFileMount + VolumeInitScripts []VolumeInitScript FSGroup *int64 VolumeOwnershipFixes []VolumeOwnershipFix SHMSizeGB int @@ -88,6 +89,12 @@ type VolumeOwnershipFix struct { GID int64 } +type VolumeInitScript struct { + Name string + MountPath string + Script string +} + // CreatePod creates a new pod for an instance func (s *PodService) CreatePod(ctx context.Context, config PodConfig) (*corev1.Pod, error) { if s.client == nil { @@ -263,6 +270,13 @@ func (s *PodService) CreatePod(ctx context.Context, config PodConfig) (*corev1.P }) } + for index, initScript := range config.VolumeInitScripts { + if initScript.Name == "" || initScript.MountPath == "" || initScript.Script == "" { + continue + } + pod.Spec.InitContainers = append(pod.Spec.InitContainers, buildVolumeInitScriptContainer(index, config.Image, pullPolicy, initScript)) + } + for index, fix := range config.VolumeOwnershipFixes { if fix.Name == "" || fix.MountPath == "" || fix.UID < 0 || fix.GID < 0 { continue @@ -428,6 +442,33 @@ fi`, } } +func buildVolumeInitScriptContainer(index int, image string, pullPolicy corev1.PullPolicy, initScript VolumeInitScript) corev1.Container { + rootUser := int64(0) + return corev1.Container{ + Name: fmt.Sprintf("init-volume-layout-%d", index+1), + Image: image, + ImagePullPolicy: pullPolicy, + Command: []string{ + "sh", + "-c", + initScript.Script, + }, + Env: []corev1.EnvVar{ + {Name: "CLAWMANAGER_VOLUME_PATH", Value: initScript.MountPath}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &rootUser, + RunAsGroup: &rootUser, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: initScript.Name, + MountPath: initScript.MountPath, + }, + }, + } +} + func normalizePodRuntimeType(runtimeType string) string { if runtimeType == "shell" { return "shell" diff --git a/backend/internal/services/k8s/pvc_service.go b/backend/internal/services/k8s/pvc_service.go index cf2b62e..9f8ec0e 100644 --- a/backend/internal/services/k8s/pvc_service.go +++ b/backend/internal/services/k8s/pvc_service.go @@ -319,7 +319,7 @@ func (s *PVCService) hostPathPVNodeAffinity(ctx context.Context) (*corev1.Volume } candidates := []candidate{} for _, node := range nodes.Items { - if node.Spec.Unschedulable || !isStorageNodeReady(node) { + if !isHostPathPVNodeCandidate(node) { continue } hostname := strings.TrimSpace(node.Labels["kubernetes.io/hostname"]) @@ -367,6 +367,18 @@ func isStorageNodeReady(node corev1.Node) bool { return false } +func isHostPathPVNodeCandidate(node corev1.Node) bool { + if node.Spec.Unschedulable || !isStorageNodeReady(node) { + return false + } + for _, taint := range node.Spec.Taints { + if taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute { + return false + } + } + return true +} + func (s *PVCService) monitorPVCBinding(ctx context.Context, namespace, pvcName string, userID, instanceID, storageSizeGB int, storageClass string, timeout time.Duration) { if _, err := s.waitForPVCBinding(ctx, namespace, pvcName, userID, instanceID, storageSizeGB, storageClass, timeout); err != nil { fmt.Printf("Async PVC binding monitor failed for %s: %v\n", pvcName, err) @@ -467,6 +479,10 @@ func (s *PVCService) createPVForPVC(ctx context.Context, namespace, pvcName stri } storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB)) + nodeAffinity, err := s.hostPathPVNodeAffinity(ctx) + if err != nil { + return nil, fmt.Errorf("failed to select node for hostPath PV %s: %w", pvName, err) + } pv := &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ @@ -487,6 +503,7 @@ func (s *PVCService) createPVForPVC(ctx context.Context, namespace, pvcName stri }, PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, StorageClassName: storageClass, + NodeAffinity: nodeAffinity, PersistentVolumeSource: corev1.PersistentVolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: hostPath, diff --git a/backend/internal/services/k8s/pvc_service_test.go b/backend/internal/services/k8s/pvc_service_test.go index ace8c3d..0965961 100644 --- a/backend/internal/services/k8s/pvc_service_test.go +++ b/backend/internal/services/k8s/pvc_service_test.go @@ -105,6 +105,52 @@ func TestHostPathPVNodeAffinitySkipsUnschedulableAndNotReadyNodes(t *testing.T) requireHostnameAffinity(t, affinity, "node-c-host") } +func TestHostPathPVNodeAffinitySkipsHardTaintedNodes(t *testing.T) { + service := &PVCService{ + client: &Client{ + Clientset: fake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "k8s-master", + Labels: map[string]string{ + "kubernetes.io/hostname": "k8s-master", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + {Key: "node-role.kubernetes.io/control-plane", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "k8s-worker1", + Labels: map[string]string{ + "kubernetes.io/hostname": "k8s-worker1", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + }, + }, + }, + ), + }, + } + + affinity, err := service.hostPathPVNodeAffinity(context.Background()) + if err != nil { + t.Fatalf("hostPathPVNodeAffinity returned error: %v", err) + } + requireHostnameAffinity(t, affinity, "k8s-worker1") +} + func requireHostnameAffinity(t *testing.T, affinity *corev1.VolumeNodeAffinity, hostname string) { t.Helper() if affinity == nil || affinity.Required == nil || len(affinity.Required.NodeSelectorTerms) != 1 { diff --git a/deployments/k3s/clawmanager.yaml b/deployments/k3s/clawmanager.yaml index 08c1503..7c72dac 100644 --- a/deployments/k3s/clawmanager.yaml +++ b/deployments/k3s/clawmanager.yaml @@ -664,6 +664,13 @@ data: UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 023_normalize_runtime_config_mounts.sql: | + USE clawmanager; + UPDATE instances + SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP + WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); --- apiVersion: v1 kind: PersistentVolumeClaim diff --git a/deployments/k8s/clawmanager.yaml b/deployments/k8s/clawmanager.yaml index e99a2f7..27ac892 100644 --- a/deployments/k8s/clawmanager.yaml +++ b/deployments/k8s/clawmanager.yaml @@ -668,6 +668,13 @@ data: UNIQUE KEY uk_openclaw_bundle_skill (bundle_id, skill_id), INDEX idx_openclaw_bundle_skill_bundle (bundle_id, sort_order) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + 023_normalize_runtime_config_mounts.sql: | + USE clawmanager; + UPDATE instances + SET mount_path = '/config', + updated_at = CURRENT_TIMESTAMP + WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') + AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); --- apiVersion: v1 kind: PersistentVolume diff --git a/docs/hermes-runtime-agent-development.md b/docs/hermes-runtime-agent-development.md index a8de538..1fc396a 100644 --- a/docs/hermes-runtime-agent-development.md +++ b/docs/hermes-runtime-agent-development.md @@ -12,11 +12,12 @@ A Hermes image must satisfy two layers of requirements: Current Hermes runtime defaults in ClawManager: - Port: `3001` +- PVC mount path: `/config` - Persistent directory: `/config/.hermes` - Default title: `Hermes Runtime` - Proxy path: ClawManager rewrites `SUBFOLDER` to `/api/v1/instances/{instance_id}/proxy/` when the instance is created. -Do not change the port or persistent directory in the image. If the image uses a different port or mount path, instance proxying, PVC mounting, and user data persistence will no longer match ClawManager expectations. +Do not change the port, PVC mount path, or persistent directory in the image. ClawManager mounts the instance PVC at `/config` so Webtop desktop files such as `/config/Desktop` and Hermes runtime files under `/config/.hermes` persist together. ## Image Build Requirements @@ -799,7 +800,7 @@ Never commit real tokens, channel secrets, Gateway API keys, or downloaded sessi Before delivering a Hermes image, verify: - The Webtop desktop is reachable through the ClawManager instance proxy. -- `/config/.hermes` is mounted and persists across restarts. +- `/config` is mounted and both `/config/Desktop` and `/config/.hermes` persist across restarts. - The agent registers and starts heartbeat within 30 seconds. - The instance detail page shows agent online, runtime running, and an updated last report time. - CPU, memory, disk, and network metrics are visible and refresh continuously. @@ -819,7 +820,7 @@ ClawManager must keep the following capabilities for Hermes to work end to end: - Allow `hermes` instances to register with the Agent Control Plane. - Inject `CLAWMANAGER_LLM_*` and OpenAI-compatible variables so Hermes can access models through ClawManager AI Gateway. - Inject Hermes and generic runtime bootstrap variables for channels, skills, and related resources. -- Mount persistent storage at `/config/.hermes`. +- Mount persistent storage at `/config`, while keeping Hermes runtime state under `/config/.hermes`. - Support `.hermes` import and export. - Keep compatibility fields such as `openclaw_status`, `openclaw_pid`, and `openclaw_version` until generic runtime fields are introduced. - Add Hermes-specific runtime control commands, or generic `start_runtime`, `stop_runtime`, and `restart_runtime`, if runtime process control becomes required.