mirrors/yuan-lab-llm--clawmanager
1
0
Fork 0
Code Issues Pull Requests Actions 1 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: support lite/pro runtime modes and runtime rollout

Browse Source
This commit is contained in:
Qingshan Chen
2026-06-14 14:30:47 +08:00
parent 22b7868b4a
commit aea224426f
197 changed files with 34057 additions and 4456 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yml
+8
View File
@@ -8,6 +8,7 @@ on:
- 'feature/**'
- 'fix/**'
pull_request:
workflow_dispatch:
permissions:
contents: read
@@ -106,6 +107,7 @@ jobs:
uses: helm/kind-action@v1.12.0
with:
cluster_name: clawmanager-ci
ignore_failed_clean: true
- name: Load image into kind
run: kind load docker-image clawmanager:ci --name clawmanager-ci
@@ -157,6 +159,12 @@ jobs:
count=1,
)
text = re.sub(
r"(\n\s+- name: workspaces\s+)nfs:\s+server: workspace-store\.clawmanager-system\.svc\.cluster\.local\s+path: /exports/workspaces",
r"\1emptyDir: {}",
text,
)
path.write_text(text, encoding="utf-8")
PY
.gitignore
+3
View File
@@ -59,6 +59,9 @@ Thumbs.db
.codex-logs/
.cache/
.VSCodeCounter/
.superpowers/
dev_docs/
devplan.md
AGENTS.md
backend/cmd/server/main.go
+127 -3
View File
@@ -6,6 +6,7 @@ import (
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
@@ -14,6 +15,7 @@ import (
"clawreef/internal/db"
"clawreef/internal/handlers"
"clawreef/internal/middleware"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services"
"clawreef/internal/services/k8s"
@@ -65,9 +67,14 @@ func main() {
instanceDesiredStateRepo := repository.NewInstanceDesiredStateRepository(database)
instanceCommandRepo := repository.NewInstanceCommandRepository(database)
instanceConfigRevisionRepo := repository.NewInstanceConfigRevisionRepository(database)
runtimePodRepo := repository.NewRuntimePodRepository(database)
bindingRepo := repository.NewInstanceRuntimeBindingRepository(database)
rolloutRepo := repository.NewRuntimeRolloutRepository(database)
workspaceFileAuditRepo := repository.NewWorkspaceFileAuditRepository(database)
teamRepo := repository.NewTeamRepository(database)
skillRepo := repository.NewSkillRepository(database)
securityScanRepo := repository.NewSecurityScanRepository(database)
instanceExternalAccessRepo := repository.NewInstanceExternalAccessRepository(database)
if repaired, repairErr := services.RepairSeededAdminPassword(userRepo); repairErr != nil {
log.Printf("Warning: failed to repair seeded admin password: %v", repairErr)
@@ -98,26 +105,53 @@ func main() {
aiObservabilityService := services.NewAIObservabilityService(modelInvocationRepo, auditEventRepo, costRecordRepo, riskHitRepo, chatMessageRepo, llmModelRepo, instanceRepo, userRepo)
clusterResourceService := services.NewClusterResourceService(instanceRepo)
services.SetRuntimeImageSettingsProvider(systemImageSettingService)
services.SetOpenClawTransferRuntimeRepositories(instanceRepo, bindingRepo, runtimePodRepo)
runtimeAgentClient := services.NewRuntimeAgentClient(cfg.Runtime.AgentControlToken)
instanceService := services.NewInstanceService(
instanceRepo,
quotaRepo,
llmModelRepo,
openClawConfigService,
services.WithPrivilegedInstancePods(cfg.Kubernetes.Runtime.Pod.Privileged),
services.WithV2RuntimeLifecycle(runtimePodRepo, bindingRepo, runtimeAgentClient, cfg.Runtime.WorkspaceRoot),
)
instanceAgentService := services.NewInstanceAgentService(instanceRepo, instanceAgentRepo, instanceDesiredStateRepo, instanceRuntimeStatusRepo, instanceCommandRepo)
instanceRuntimeStatusService := services.NewInstanceRuntimeStatusService(instanceRuntimeStatusRepo, instanceAgentRepo, instanceDesiredStateRepo)
instanceCommandService := services.NewInstanceCommandService(instanceCommandRepo, instanceRuntimeStatusRepo, instanceDesiredStateRepo)
instanceConfigRevisionService := services.NewInstanceConfigRevisionService(instanceConfigRevisionRepo)
teamService := services.NewTeamService(teamRepo, instanceService)
teamService := services.NewTeamService(teamRepo, instanceService, services.WithTeamRuntimeWorkspaceRoot(cfg.Runtime.WorkspaceRoot))
var platformRedis services.PlatformRedisClient
if redisURL := strings.TrimSpace(cfg.Runtime.RedisURL); redisURL != "" {
var redisErr error
platformRedis, redisErr = services.NewPlatformRedisClient(redisURL)
if redisErr != nil {
log.Printf("platform redis disabled: %v", redisErr)
}
} else {
log.Printf("platform redis disabled: redis url is empty")
}
runtimeEvents := services.NewRuntimeEventService(platformRedis)
workspaceFileService := services.NewWorkspaceFileService(workspaceFileAuditRepo)
runtimeWorkspaceFileService := services.NewRuntimeWorkspaceFileService(workspaceFileAuditRepo)
skillService := services.NewSkillService(skillRepo, instanceRepo, instanceCommandService, objectStorageService, skillScannerClient)
securityScanService := services.NewSecurityScanService(securityScanRepo, skillRepo, objectStorageService, skillScannerClient)
externalAccessService := services.NewInstanceExternalAccessService(instanceExternalAccessRepo)
aiGatewayService := aigateway.NewService(llmModelRepo, modelInvocationService, auditEventService, costRecordService, riskDetectionService, riskHitService, chatSessionService, chatMessageService)
// Initialize handlers
authHandler := handlers.NewAuthHandler(authService)
userHandler := handlers.NewUserHandler(userService, quotaService)
instanceHandler := handlers.NewInstanceHandler(instanceService, instanceAgentService, instanceRuntimeStatusService, instanceCommandService, instanceConfigRevisionService, openClawConfigService, skillService)
instanceHandler := handlers.NewInstanceHandler(
instanceService,
instanceAgentService,
instanceRuntimeStatusService,
instanceCommandService,
instanceConfigRevisionService,
openClawConfigService,
skillService,
externalAccessService,
services.WithInstanceProxyRuntimeRepositories(instanceRepo, runtimePodRepo, bindingRepo),
)
systemSettingsHandler := handlers.NewSystemSettingsHandler(systemImageSettingService)
llmModelHandler := handlers.NewLLMModelHandler(llmModelService)
aiGatewayHandler := handlers.NewAIGatewayHandler(aiGatewayService)
@@ -130,15 +164,65 @@ func main() {
securityHandler := handlers.NewSecurityHandler(securityScanService)
agentHandler := handlers.NewAgentHandler(instanceAgentService, instanceCommandService, instanceRuntimeStatusService, instanceConfigRevisionService, skillService)
teamHandler := handlers.NewTeamHandler(teamService)
workspaceFileHandler := handlers.NewWorkspaceFileHandler(instanceService, workspaceFileService, runtimeWorkspaceFileService)
runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, runtimeEvents)
// Initialize WebSocket hub and handler
wsHub := services.GetHub()
wsHandler := handlers.NewWebSocketHandler(wsHub)
var runtimeAdminEventBridgeCancel context.CancelFunc
if platformRedis != nil {
var bridgeCtx context.Context
bridgeCtx, runtimeAdminEventBridgeCancel = context.WithCancel(context.Background())
services.StartRuntimeAdminEventBridge(bridgeCtx, runtimeEvents, wsHub)
}
// Start sync service to keep instance status in sync with K8s
syncService := services.NewSyncService(instanceRepo, instanceRuntimeStatusService)
syncService.Start()
teamService.Start()
var runtimeSchedulerCancel context.CancelFunc
var runtimeScheduler *services.RuntimeScheduler
if cfg.Runtime.SchedulerEnabled {
k8sClient := k8s.GetClient()
if k8sClient == nil || k8sClient.Clientset == nil {
log.Printf("runtime scheduler disabled: k8s client is unavailable")
} else {
leader := services.NewRuntimeLeaderService(k8sClient.Clientset, cfg.Runtime.Namespace, cfg.Runtime.BackendReplicaID)
runtimeDeployments := k8s.NewRuntimeDeploymentService(k8sClient.Clientset)
runtimeSchedulerOptions := []services.RuntimeSchedulerOption{
services.WithRuntimeSchedulerWorkspaceRoot(cfg.Runtime.WorkspaceRoot),
services.WithRuntimeSchedulerNamespace(cfg.Runtime.Namespace),
services.WithRuntimeSchedulerGatewayPortRange(cfg.Runtime.GatewayPortStart, cfg.Runtime.GatewayPortEnd),
services.WithRuntimeSchedulerHeartbeatTimeout(cfg.Runtime.HeartbeatTimeout),
services.WithRuntimeSchedulerMaxGatewaysPerPod(cfg.Runtime.MaxGatewaysPerPod),
}
if gatewayEnvProvider, ok := instanceService.(interface {
BuildGatewayEnv(*models.Instance) (map[string]string, error)
}); ok {
runtimeSchedulerOptions = append(runtimeSchedulerOptions, services.WithRuntimeSchedulerGatewayEnvBuilder(gatewayEnvProvider.BuildGatewayEnv))
}
runtimeScheduler = services.NewRuntimeScheduler(
instanceRepo,
runtimePodRepo,
bindingRepo,
rolloutRepo,
runtimeAgentClient,
runtimeEvents,
leader,
runtimeDeployments,
cfg.Runtime.SchedulerTick,
runtimeSchedulerOptions...,
)
var schedulerCtx context.Context
schedulerCtx, runtimeSchedulerCancel = context.WithCancel(context.Background())
runtimeScheduler.Start(schedulerCtx)
log.Printf("runtime scheduler started")
}
} else {
log.Printf("runtime scheduler disabled by configuration")
}
runtimePoolHandler := handlers.NewRuntimePoolHandler(runtimePodRepo, bindingRepo, rolloutRepo, runtimeScheduler, runtimeEvents)
// Setup router
r := gin.Default()
@@ -150,8 +234,20 @@ func main() {
r.NoMethod(egressProxyHandler.Handle)
// Routes
r.Any("/s/:code", instanceHandler.OpenShortExternalAccess)
r.Any("/s/:code/*path", instanceHandler.OpenShortExternalAccess)
api := r.Group("/api/v1")
{
runtimeAgent := api.Group("/runtime-agent")
{
runtimeAgent.POST("/register", runtimeAgentHandler.Register)
runtimeAgent.POST("/heartbeat", runtimeAgentHandler.Heartbeat)
runtimeAgent.POST("/metrics/report", runtimeAgentHandler.ReportMetrics)
runtimeAgent.POST("/gateways/report", runtimeAgentHandler.ReportGateways)
runtimeAgent.POST("/skills/report", runtimeAgentHandler.ReportSkills)
}
// Auth routes
auth := api.Group("/auth")
{
@@ -212,6 +308,17 @@ func main() {
instances.POST("/:id/openclaw/import", instanceHandler.ImportOpenClaw)
instances.GET("/:id/hermes/export", instanceHandler.ExportHermes)
instances.POST("/:id/hermes/import", instanceHandler.ImportHermes)
instances.GET("/:id/external-access", instanceHandler.GetExternalAccess)
instances.POST("/:id/external-access/share-link", instanceHandler.EnableShareLink)
instances.POST("/:id/external-access/password", instanceHandler.CreateExternalAccessPassword)
instances.DELETE("/:id/external-access", instanceHandler.DisableExternalAccess)
instances.GET("/:id/workspace/files", workspaceFileHandler.List)
instances.GET("/:id/workspace/preview", workspaceFileHandler.Preview)
instances.GET("/:id/workspace/download", workspaceFileHandler.Download)
instances.POST("/:id/workspace/upload", workspaceFileHandler.Upload)
instances.POST("/:id/workspace/folders", workspaceFileHandler.Mkdir)
instances.PATCH("/:id/workspace/entries", workspaceFileHandler.Rename)
instances.DELETE("/:id/workspace/entries", workspaceFileHandler.Delete)
instances.GET("/:id/skills", skillHandler.ListInstanceSkills)
instances.POST("/:id/skills", skillHandler.AttachSkillToInstance)
instances.DELETE("/:id/skills/:skillId", skillHandler.RemoveSkillFromInstance)
@@ -229,6 +336,17 @@ func main() {
adminInstances.GET("", instanceHandler.ListAllInstances)
}
adminRuntime := api.Group("/admin")
adminRuntime.Use(middleware.Auth())
adminRuntime.Use(middleware.SetUserInfo(userRepo))
adminRuntime.Use(middleware.NewAdminAuth(userRepo))
{
adminRuntime.GET("/runtime-pods", runtimePoolHandler.ListPods)
adminRuntime.GET("/runtime-pods/:id/gateways", runtimePoolHandler.GetPodGateways)
adminRuntime.POST("/runtime-pods/:id/drain", runtimePoolHandler.DrainPod)
adminRuntime.POST("/runtime-rollouts", runtimePoolHandler.StartRollout)
}
teams := api.Group("/teams")
teams.Use(middleware.Auth())
teams.Use(middleware.SetUserInfo(userRepo))
@@ -360,7 +478,7 @@ func main() {
}
gatewayLLM := api.Group("/gateway/llm")
gatewayLLM.Use(middleware.GatewayAuth(instanceRepo))
gatewayLLM.Use(middleware.GatewayAuth(instanceRepo, bindingRepo))
{
gatewayLLM.GET("/models", aiGatewayHandler.ListModels)
gatewayLLM.POST("/chat/completions", aiGatewayHandler.ChatCompletions)
@@ -423,6 +541,12 @@ func main() {
}
// Stop background services
if runtimeSchedulerCancel != nil {
runtimeSchedulerCancel()
}
if runtimeAdminEventBridgeCancel != nil {
runtimeAdminEventBridgeCancel()
}
syncService.Stop()
teamService.Stop()
wsHub.Stop()
backend/deployments/k8s/clawreef-incluster.yaml
+35 -2
View File
@@ -413,8 +413,8 @@ data:
SELECT
'openclaw',
'shell',
'OpenClaw Shell',
'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest',
'OpenClaw Lite',
'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
@@ -449,6 +449,39 @@ data:
updated_at = CURRENT_TIMESTAMP
WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes')
AND mount_path IN ('/data', '/home/user/data', '/config/.hermes');
031_add_hermes_lite_runtime_image.sql: |
USE clawreef;
UPDATE system_image_settings
SET
image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
display_name = 'Hermes Lite',
is_enabled = TRUE
WHERE instance_type = 'hermes'
AND runtime_type = 'shell'
AND image IN (
'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest',
'registry.example.com/your-custom-shell-image:latest'
);
INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
SELECT
'hermes',
'shell',
'Hermes Lite',
'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
FROM system_image_settings
WHERE instance_type = 'hermes'
AND runtime_type = 'shell'
)
AND NOT EXISTS (
SELECT 1
FROM system_image_settings
WHERE instance_type = 'hermes'
AND is_enabled = FALSE
);
---
apiVersion: v1
kind: PersistentVolumeClaim
backend/internal/aigateway/service.go
+124 -2
View File
@@ -72,6 +72,10 @@ type ChatCompletionRequest struct {
User *string `json:"user,omitempty"`
SessionID *string `json:"session_id,omitempty"`
InstanceID *int `json:"instance_id,omitempty"`
InstanceMode *string `json:"instance_mode,omitempty"`
RuntimeType *string `json:"runtime_type,omitempty"`
GatewayID *string `json:"gateway_id,omitempty"`
RuntimePodID *int64 `json:"runtime_pod_id,omitempty"`
TraceID *string `json:"trace_id,omitempty"`
RequestID *string `json:"request_id,omitempty"`
}
@@ -357,6 +361,10 @@ func (s *service) ChatCompletions(ctx context.Context, userID int, req ChatCompl
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.request.blocked",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityWarn,
@@ -388,6 +396,10 @@ func (s *service) StreamChatCompletions(ctx context.Context, userID int, req Cha
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.request.blocked",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityWarn,
@@ -449,6 +461,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.request.received",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityInfo,
@@ -465,6 +481,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.risk.detected",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityWarn,
@@ -477,7 +497,7 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
resolvedModel, riskAction, resolveErr := s.resolveTargetModel(prepared.selectedModel, riskAnalysis)
if resolveErr != nil {
blockedInvocationID := s.recordBlockedInvocation(prepared.traceID, prepared.requestID, prepared.req, prepared.userID, prepared.selectedModel, resolveErr.Error())
if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, blockedInvocationID, riskAction, riskAnalysis.Hits); err != nil {
if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, blockedInvocationID, riskHitAttribution(prepared.req), riskAction, riskAnalysis.Hits); err != nil {
logPersistenceError("record blocked risk hits", prepared.traceID, err)
}
if err := s.auditEventService.RecordEvent(&models.AuditEvent{
@@ -486,6 +506,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
InvocationID: blockedInvocationID,
EventType: "gateway.request.blocked",
TrafficClass: models.TrafficClassLLM,
@@ -498,7 +522,7 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
}
if riskAnalysis.IsSensitive {
if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, nil, riskAction, riskAnalysis.Hits); err != nil {
if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, nil, riskHitAttribution(prepared.req), riskAction, riskAnalysis.Hits); err != nil {
logPersistenceError("record risk hits", prepared.traceID, err)
}
if riskAction == models.RiskActionRouteSecureModel && resolvedModel != nil && resolvedModel.ID != prepared.selectedModel.ID {
@@ -508,6 +532,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.request.rerouted",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityWarn,
@@ -655,6 +683,10 @@ func (s *service) recordFailure(traceID, requestID string, req ChatCompletionReq
RequestID: requestID,
UserID: intPtr(userID),
InstanceID: req.InstanceID,
InstanceMode: runtimeAttributionString(req.InstanceMode),
RuntimeType: runtimeAttributionString(req.RuntimeType),
GatewayID: runtimeAttributionString(req.GatewayID),
RuntimePodID: runtimeAttributionInt64(req.RuntimePodID),
ModelID: intPtr(model.ID),
ProviderType: model.ProviderType,
RequestedModel: req.Model,
@@ -679,6 +711,10 @@ func (s *service) recordFailure(traceID, requestID string, req ChatCompletionReq
RequestID: stringPtr(requestID),
UserID: intPtr(userID),
InstanceID: req.InstanceID,
InstanceMode: runtimeAttributionString(req.InstanceMode),
RuntimeType: runtimeAttributionString(req.RuntimeType),
GatewayID: runtimeAttributionString(req.GatewayID),
RuntimePodID: runtimeAttributionInt64(req.RuntimePodID),
InvocationID: intPtr(invocation.ID),
EventType: "gateway.request.failed",
TrafficClass: models.TrafficClassLLM,
@@ -703,6 +739,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo
RequestID: prepared.requestID,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
ModelID: intPtr(prepared.resolvedModel.ID),
ProviderType: prepared.resolvedModel.ProviderType,
RequestedModel: prepared.req.Model,
@@ -750,6 +790,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
InvocationID: intPtr(invocation.ID),
ModelID: intPtr(prepared.resolvedModel.ID),
ProviderType: prepared.resolvedModel.ProviderType,
@@ -772,6 +816,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
InvocationID: intPtr(invocation.ID),
EventType: "gateway.request.completed",
TrafficClass: models.TrafficClassLLM,
@@ -787,6 +835,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
GatewayID: runtimeAttributionString(prepared.req.GatewayID),
RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
InvocationID: intPtr(invocation.ID),
EventType: "gateway.usage.estimated",
TrafficClass: models.TrafficClassLLM,
@@ -1312,6 +1364,10 @@ func buildOpenAICompatibleRequestBody(req ChatCompletionRequest, model *models.L
}
delete(payload, "session_id")
delete(payload, "instance_id")
delete(payload, "instance_mode")
delete(payload, "runtime_type")
delete(payload, "gateway_id")
delete(payload, "runtime_pod_id")
delete(payload, "trace_id")
delete(payload, "request_id")
modelPayload, err := json.Marshal(model.ProviderModelName)
@@ -1847,6 +1903,10 @@ func (s *service) recordBlockedInvocation(traceID, requestID string, req ChatCom
RequestID: requestID,
UserID: intPtr(userID),
InstanceID: req.InstanceID,
InstanceMode: runtimeAttributionString(req.InstanceMode),
RuntimeType: runtimeAttributionString(req.RuntimeType),
GatewayID: runtimeAttributionString(req.GatewayID),
RuntimePodID: runtimeAttributionInt64(req.RuntimePodID),
ModelID: intPtr(model.ID),
ProviderType: model.ProviderType,
RequestedModel: req.Model,
@@ -2042,6 +2102,68 @@ func intPtr(value int) *int {
return &value
}
func runtimeAttributionString(value *string) *string {
if value == nil {
return nil
}
trimmed := strings.TrimSpace(*value)
if trimmed == "" {
return nil
}
return &trimmed
}
func runtimeAttributionInt64(value *int64) *int64 {
if value == nil || *value <= 0 {
return nil
}
copied := *value
return &copied
}
func applyInvocationRuntimeAttribution(invocation *models.ModelInvocation, req ChatCompletionRequest) {
if invocation == nil {
return
}
invocation.InstanceMode = runtimeAttributionString(req.InstanceMode)
invocation.RuntimeType = runtimeAttributionString(req.RuntimeType)
invocation.GatewayID = runtimeAttributionString(req.GatewayID)
invocation.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID)
}
func applyAuditRuntimeAttribution(event *models.AuditEvent, req ChatCompletionRequest) {
if event == nil {
return
}
event.InstanceMode = runtimeAttributionString(req.InstanceMode)
event.RuntimeType = runtimeAttributionString(req.RuntimeType)
event.GatewayID = runtimeAttributionString(req.GatewayID)
event.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID)
}
func applyCostRuntimeAttribution(record *models.CostRecord, req ChatCompletionRequest) {
if record == nil {
return
}
record.InstanceMode = runtimeAttributionString(req.InstanceMode)
record.RuntimeType = runtimeAttributionString(req.RuntimeType)
record.GatewayID = runtimeAttributionString(req.GatewayID)
record.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID)
}
func riskHitAttribution(req ChatCompletionRequest) *services.RiskHitAttribution {
attribution := &services.RiskHitAttribution{
InstanceMode: runtimeAttributionString(req.InstanceMode),
RuntimeType: runtimeAttributionString(req.RuntimeType),
GatewayID: runtimeAttributionString(req.GatewayID),
RuntimePodID: runtimeAttributionInt64(req.RuntimePodID),
}
if attribution.InstanceMode == nil && attribution.RuntimeType == nil && attribution.GatewayID == nil && attribution.RuntimePodID == nil {
return nil
}
return attribution
}
func userIDOrZero(value *int) int {
if value == nil {
return 0
backend/internal/aigateway/service_test.go
+55 -1
View File
@@ -47,7 +47,7 @@ func (s *stubChatSessionService) EnsureSession(sessionID string, userID, instanc
func TestBuildProviderRequestPreservesToolConfiguration(t *testing.T) {
req := ChatCompletionRequest{
Model: "gateway-model",
RawBody: []byte(`{"model":"gateway-model","messages":[{"role":"user","content":[{"type":"text","text":"weather in shanghai"}]}],"tools":[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"get_weather"}},"stream":true,"stream_options":{"include_usage":false,"custom":"value"},"custom_field":{"nested":[1,2,3]},"session_id":"sess_123","request_id":"req_123","trace_id":"trc_123","instance_id":99}`),
RawBody: []byte(`{"model":"gateway-model","messages":[{"role":"user","content":[{"type":"text","text":"weather in shanghai"}]}],"tools":[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"get_weather"}},"stream":true,"stream_options":{"include_usage":false,"custom":"value"},"custom_field":{"nested":[1,2,3]},"session_id":"sess_123","request_id":"req_123","trace_id":"trc_123","instance_id":99,"instance_mode":"lite","runtime_type":"gateway","gateway_id":"gw_123","runtime_pod_id":5}`),
}
model := &models.LLMModel{
@@ -95,6 +95,18 @@ func TestBuildProviderRequestPreservesToolConfiguration(t *testing.T) {
if _, ok := payload["instance_id"]; ok {
t.Fatalf("expected internal instance_id to be stripped")
}
if _, ok := payload["instance_mode"]; ok {
t.Fatalf("expected internal instance_mode to be stripped")
}
if _, ok := payload["runtime_type"]; ok {
t.Fatalf("expected internal runtime_type to be stripped")
}
if _, ok := payload["gateway_id"]; ok {
t.Fatalf("expected internal gateway_id to be stripped")
}
if _, ok := payload["runtime_pod_id"]; ok {
t.Fatalf("expected internal runtime_pod_id to be stripped")
}
if string(payload["stream_options"]) != `{"include_usage":false,"custom":"value"}` {
t.Fatalf("expected stream_options to pass through unchanged, got %s", string(payload["stream_options"]))
}
@@ -135,6 +147,48 @@ func TestBuildProviderRequestUsesAnthropicProtocolForLocalModel(t *testing.T) {
}
}
func TestRuntimeAttributionHelpersPopulateRecords(t *testing.T) {
mode := "lite"
runtimeType := "gateway"
gatewayID := "gw_instance_1"
runtimePodID := int64(17)
req := ChatCompletionRequest{
InstanceMode: &mode,
RuntimeType: &runtimeType,
GatewayID: &gatewayID,
RuntimePodID: &runtimePodID,
}
invocation := &models.ModelInvocation{}
applyInvocationRuntimeAttribution(invocation, req)
if invocation.InstanceMode == nil || *invocation.InstanceMode != mode {
t.Fatalf("invocation instance mode not populated")
}
if invocation.GatewayID == nil || *invocation.GatewayID != gatewayID {
t.Fatalf("invocation gateway ID not populated")
}
audit := &models.AuditEvent{}
applyAuditRuntimeAttribution(audit, req)
if audit.RuntimeType == nil || *audit.RuntimeType != runtimeType {
t.Fatalf("audit runtime type not populated")
}
if audit.RuntimePodID == nil || *audit.RuntimePodID != runtimePodID {
t.Fatalf("audit runtime pod ID not populated")
}
cost := &models.CostRecord{}
applyCostRuntimeAttribution(cost, req)
if cost.GatewayID == nil || *cost.GatewayID != gatewayID {
t.Fatalf("cost gateway ID not populated")
}
risk := riskHitAttribution(req)
if risk == nil || risk.GatewayID == nil || *risk.GatewayID != gatewayID {
t.Fatalf("risk attribution gateway ID not populated")
}
}
func TestRewriteStreamLineKeepsToolCalls(t *testing.T) {
line := "data: {\"id\":\"chatcmpl-1\",\"model\":\"provider-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Shanghai\\\"}\"}}]},\"finish_reason\":null}]}\n"
backend/internal/config/config.go
+125 -16
View File
@@ -3,7 +3,9 @@ package config
import (
"fmt"
"os"
"strconv"
"strings"
"time"
"gopkg.in/yaml.v3"
)
@@ -14,6 +16,7 @@ type Config struct {
Database DatabaseConfig `yaml:"database"`
JWT JWTConfig `yaml:"jwt"`
Kubernetes KubernetesConfig `yaml:"kubernetes"`
Runtime RuntimePoolConfig `yaml:"runtime"`
ObjectStorage ObjectStorageConfig `yaml:"objectStorage"`
SkillScanner SkillScannerConfig `yaml:"skillScanner"`
}
@@ -42,12 +45,12 @@ type JWTConfig struct {
// KubernetesConfig holds Kubernetes-related configuration
type KubernetesConfig struct {
Mode string `yaml:"mode"` // 连接模式: auto, incluster, outofcluster
OutOfCluster OutOfClusterConfig `yaml:"outOfCluster"`
InCluster InClusterConfig `yaml:"inCluster"`
Common CommonKubernetesConfig `yaml:"common"`
Runtime RuntimeConfig `yaml:"runtime"`
Logging LoggingConfig `yaml:"logging"`
Mode string `yaml:"mode"` // 连接模式: auto, incluster, outofcluster
OutOfCluster OutOfClusterConfig `yaml:"outOfCluster"`
InCluster InClusterConfig `yaml:"inCluster"`
Common CommonKubernetesConfig `yaml:"common"`
Runtime KubernetesRuntimeConfig `yaml:"runtime"`
Logging LoggingConfig `yaml:"logging"`
}
// OutOfClusterConfig holds out-of-cluster Kubernetes configuration
@@ -82,8 +85,8 @@ type CommonKubernetesConfig struct {
AutoCreateNamespace bool `yaml:"autoCreateNamespace"`
}
// RuntimeConfig holds runtime configuration
type RuntimeConfig struct {
// KubernetesRuntimeConfig holds Kubernetes runtime configuration
type KubernetesRuntimeConfig struct {
Pod RuntimePodConfig `yaml:"pod"`
PVC RuntimePVCConfig `yaml:"pvc"`
}
@@ -116,6 +119,26 @@ type RuntimePVCConfig struct {
HostPathPrefix string `yaml:"hostPathPrefix"`
}
// RuntimePoolConfig holds shared V2 runtime pool configuration.
type RuntimePoolConfig struct {
Namespace string `yaml:"namespace"`
WorkspaceRoot string `yaml:"workspaceRoot"`
WorkspaceNFSServer string `yaml:"workspaceNfsServer"`
WorkspaceNFSPath string `yaml:"workspaceNfsPath"`
AgentControlToken string `yaml:"agentControlToken"`
AgentReportToken string `yaml:"agentReportToken"`
BackendReplicaID string `yaml:"backendReplicaId"`
RedisURL string `yaml:"redisUrl"`
SchedulerEnabled bool `yaml:"schedulerEnabled"`
HeartbeatTimeout time.Duration `yaml:"heartbeatTimeout"`
SchedulerTick time.Duration `yaml:"schedulerTick"`
OpenClawImage string `yaml:"openClawImage"`
HermesImage string `yaml:"hermesImage"`
MaxGatewaysPerPod int `yaml:"maxGatewaysPerPod"`
GatewayPortStart int `yaml:"gatewayPortStart"`
GatewayPortEnd int `yaml:"gatewayPortEnd"`
}
// LoggingConfig holds logging configuration
type LoggingConfig struct {
Level string `yaml:"level"`
@@ -135,14 +158,15 @@ type ObjectStorageConfig struct {
}
type SkillScannerConfig struct {
BaseURL string `yaml:"baseUrl"`
APIKey string `yaml:"apiKey"`
TimeoutSeconds int `yaml:"timeoutSeconds"`
Enabled bool `yaml:"enabled"`
BaseURL string `yaml:"baseUrl"`
APIKey string `yaml:"apiKey"`
TimeoutSeconds int `yaml:"timeoutSeconds"`
Enabled bool `yaml:"enabled"`
}
// Load loads configuration from file and environment variables
func Load() (*Config, error) {
runtimeNamespace := getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", "clawmanager-system"))
config := &Config{
Server: ServerConfig{
Address: ":9001",
@@ -177,7 +201,7 @@ func Load() (*Config, error) {
RetryCount: 3,
AutoCreateNamespace: true,
},
Runtime: RuntimeConfig{
Runtime: KubernetesRuntimeConfig{
Pod: RuntimePodConfig{
ImageRegistry: "docker.io/clawreef",
ContainerPort: 3001,
@@ -199,6 +223,24 @@ func Load() (*Config, error) {
LogAPICalls: false,
},
},
Runtime: RuntimePoolConfig{
Namespace: runtimeNamespace,
WorkspaceRoot: getEnv("RUNTIME_WORKSPACE_ROOT", "/workspaces"),
WorkspaceNFSServer: getEnv("RUNTIME_WORKSPACE_NFS_SERVER", defaultWorkspaceNFSServer(runtimeNamespace)),
WorkspaceNFSPath: getEnv("RUNTIME_WORKSPACE_NFS_PATH", "/"),
AgentControlToken: getEnv("RUNTIME_AGENT_CONTROL_TOKEN", ""),
AgentReportToken: getEnv("RUNTIME_AGENT_REPORT_TOKEN", ""),
BackendReplicaID: getEnv("HOSTNAME", "clawmanager-backend-local"),
RedisURL: getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", "")),
SchedulerEnabled: getEnvBool("RUNTIME_SCHEDULER_ENABLED", true),
HeartbeatTimeout: getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", 10*time.Second),
SchedulerTick: getEnvDuration("RUNTIME_SCHEDULER_TICK", 2*time.Second),
OpenClawImage: getEnv("OPENCLAW_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest"),
HermesImage: getEnv("HERMES_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"),
MaxGatewaysPerPod: getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", 100),
GatewayPortStart: getEnvInt("RUNTIME_GATEWAY_PORT_START", 20000),
GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", 20099),
},
ObjectStorage: ObjectStorageConfig{
Endpoint: getEnv("OBJECT_STORAGE_ENDPOINT", ""),
Region: getEnv("OBJECT_STORAGE_REGION", ""),
@@ -211,10 +253,10 @@ func Load() (*Config, error) {
LocalFallback: getEnv("OBJECT_STORAGE_LOCAL_FALLBACK", ".data/object-storage"),
},
SkillScanner: SkillScannerConfig{
BaseURL: getEnv("SKILL_SCANNER_BASE_URL", ""),
APIKey: getEnv("SKILL_SCANNER_API_KEY", ""),
BaseURL: getEnv("SKILL_SCANNER_BASE_URL", ""),
APIKey: getEnv("SKILL_SCANNER_API_KEY", ""),
TimeoutSeconds: 30,
Enabled: strings.EqualFold(getEnv("SKILL_SCANNER_ENABLED", "false"), "true"),
Enabled: strings.EqualFold(getEnv("SKILL_SCANNER_ENABLED", "false"), "true"),
},
}
@@ -308,6 +350,29 @@ func applyEnvOverrides(config *Config) {
config.Kubernetes.Runtime.PVC.HostPathPrefix = hostPathPrefix
}
config.Runtime.Namespace = getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", config.Runtime.Namespace))
config.Runtime.WorkspaceRoot = getEnv("RUNTIME_WORKSPACE_ROOT", config.Runtime.WorkspaceRoot)
config.Runtime.WorkspaceNFSServer = getEnv("RUNTIME_WORKSPACE_NFS_SERVER", config.Runtime.WorkspaceNFSServer)
config.Runtime.WorkspaceNFSPath = getEnv("RUNTIME_WORKSPACE_NFS_PATH", config.Runtime.WorkspaceNFSPath)
if strings.TrimSpace(config.Runtime.WorkspaceNFSServer) == "" {
config.Runtime.WorkspaceNFSServer = defaultWorkspaceNFSServer(config.Runtime.Namespace)
}
if strings.TrimSpace(config.Runtime.WorkspaceNFSPath) == "" {
config.Runtime.WorkspaceNFSPath = "/"
}
config.Runtime.AgentControlToken = getEnv("RUNTIME_AGENT_CONTROL_TOKEN", config.Runtime.AgentControlToken)
config.Runtime.AgentReportToken = getEnv("RUNTIME_AGENT_REPORT_TOKEN", config.Runtime.AgentReportToken)
config.Runtime.BackendReplicaID = getEnv("HOSTNAME", config.Runtime.BackendReplicaID)
config.Runtime.RedisURL = getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", config.Runtime.RedisURL))
config.Runtime.SchedulerEnabled = getEnvBool("RUNTIME_SCHEDULER_ENABLED", config.Runtime.SchedulerEnabled)
config.Runtime.HeartbeatTimeout = getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", config.Runtime.HeartbeatTimeout)
config.Runtime.SchedulerTick = getEnvDuration("RUNTIME_SCHEDULER_TICK", config.Runtime.SchedulerTick)
config.Runtime.OpenClawImage = getEnv("OPENCLAW_RUNTIME_IMAGE", config.Runtime.OpenClawImage)
config.Runtime.HermesImage = getEnv("HERMES_RUNTIME_IMAGE", config.Runtime.HermesImage)
config.Runtime.MaxGatewaysPerPod = getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", config.Runtime.MaxGatewaysPerPod)
config.Runtime.GatewayPortStart = getEnvInt("RUNTIME_GATEWAY_PORT_START", config.Runtime.GatewayPortStart)
config.Runtime.GatewayPortEnd = getEnvInt("RUNTIME_GATEWAY_PORT_END", config.Runtime.GatewayPortEnd)
if endpoint := os.Getenv("OBJECT_STORAGE_ENDPOINT"); endpoint != "" {
config.ObjectStorage.Endpoint = endpoint
}
@@ -356,6 +421,50 @@ func getEnv(key, defaultValue string) string {
return defaultValue
}
func defaultWorkspaceNFSServer(namespace string) string {
namespace = strings.TrimSpace(namespace)
if namespace == "" {
namespace = "clawmanager-system"
}
return fmt.Sprintf("workspace-store.%s.svc.cluster.local", namespace)
}
func getEnvInt(key string, defaultValue int) int {
value := os.Getenv(key)
if value == "" {
return defaultValue
}
parsed, err := strconv.Atoi(value)
if err != nil {
return defaultValue
}
return parsed
}
func getEnvBool(key string, defaultValue bool) bool {
value := os.Getenv(key)
if value == "" {
return defaultValue
}
parsed, err := strconv.ParseBool(value)
if err != nil {
return defaultValue
}
return parsed
}
func getEnvDuration(key string, defaultValue time.Duration) time.Duration {
value := os.Getenv(key)
if value == "" {
return defaultValue
}
parsed, err := time.ParseDuration(value)
if err != nil {
return defaultValue
}
return parsed
}
// GetKubeconfigPath returns the kubeconfig path for out-of-cluster mode
func (c *Config) GetKubeconfigPath() string {
return c.Kubernetes.OutOfCluster.Kubeconfig
backend/internal/config/config_test.go
+29
View File
@@ -0,0 +1,29 @@
package config
import "testing"
func TestLoadRuntimeDefaults(t *testing.T) {
for _, key := range []string{
"HERMES_RUNTIME_IMAGE",
"OPENCLAW_RUNTIME_IMAGE",
"RUNTIME_NAMESPACE",
"K8S_NAMESPACE",
"HOSTNAME",
"PLATFORM_REDIS_URL",
"TEAM_REDIS_URL",
} {
t.Setenv(key, "")
}
cfg, err := Load()
if err != nil {
t.Fatalf("Load returned error: %v", err)
}
if got, want := cfg.Runtime.HermesImage, "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"; got != want {
t.Fatalf("expected Hermes default image %q, got %q", want, got)
}
if got, want := cfg.Runtime.OpenClawImage, "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest"; got != want {
t.Fatalf("expected OpenClaw default image %q, got %q", want, got)
}
}
backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql
+2 -2
View File
@@ -61,8 +61,8 @@ INSERT INTO system_image_settings (instance_type, runtime_type, display_name, im
SELECT
'openclaw',
'shell',
'OpenClaw Shell',
'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest',
'OpenClaw Lite',
'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
backend/internal/db/migrations/023_add_runtime_pool_v2.sql
+121
View File
@@ -0,0 +1,121 @@
SET @stmt = IF(
(SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'workspace_path') = 0,
'ALTER TABLE instances ADD COLUMN workspace_path VARCHAR(1024) NULL AFTER mount_path',
'SELECT 1'
);
PREPARE stmt FROM @stmt;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @stmt = IF(
(SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'workspace_usage_bytes') = 0,
'ALTER TABLE instances ADD COLUMN workspace_usage_bytes BIGINT NOT NULL DEFAULT 0 AFTER workspace_path',
'SELECT 1'
);
PREPARE stmt FROM @stmt;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @stmt = IF(
(SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'runtime_generation') = 0,
'ALTER TABLE instances ADD COLUMN runtime_generation INT NOT NULL DEFAULT 1 AFTER workspace_usage_bytes',
'SELECT 1'
);
PREPARE stmt FROM @stmt;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @stmt = IF(
(SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'runtime_error_message') = 0,
'ALTER TABLE instances ADD COLUMN runtime_error_message TEXT NULL AFTER runtime_generation',
'SELECT 1'
);
PREPARE stmt FROM @stmt;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
ALTER TABLE instances
MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop';
CREATE TABLE IF NOT EXISTS runtime_pods (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
runtime_type VARCHAR(32) NOT NULL,
namespace VARCHAR(128) NOT NULL,
pod_name VARCHAR(255) NOT NULL,
pod_uid VARCHAR(128) NULL,
pod_ip VARCHAR(64) NULL,
node_name VARCHAR(255) NULL,
deployment_name VARCHAR(255) NOT NULL,
image_ref VARCHAR(512) NOT NULL,
agent_endpoint VARCHAR(255) NULL,
state VARCHAR(32) NOT NULL DEFAULT 'pending',
capacity INT NOT NULL DEFAULT 100,
used_slots INT NOT NULL DEFAULT 0,
draining TINYINT(1) NOT NULL DEFAULT 0,
cpu_millis_used BIGINT NOT NULL DEFAULT 0,
memory_bytes_used BIGINT NOT NULL DEFAULT 0,
disk_bytes_used BIGINT NOT NULL DEFAULT 0,
network_rx_bytes BIGINT NOT NULL DEFAULT 0,
network_tx_bytes BIGINT NOT NULL DEFAULT 0,
metrics_json JSON NULL,
last_seen_at DATETIME NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_runtime_pod_namespace_name (namespace, pod_name),
KEY idx_runtime_pods_schedulable (runtime_type, state, draining, used_slots, capacity),
KEY idx_runtime_pods_last_seen (last_seen_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS instance_runtime_bindings (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
instance_id INT NOT NULL,
runtime_pod_id BIGINT NOT NULL,
runtime_type VARCHAR(32) NOT NULL,
gateway_id VARCHAR(128) NOT NULL,
gateway_port INT NOT NULL,
gateway_pid INT NULL,
workspace_path VARCHAR(1024) NOT NULL,
state VARCHAR(32) NOT NULL DEFAULT 'creating',
generation INT NOT NULL DEFAULT 1,
last_health_at DATETIME NULL,
error_message TEXT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_instance_runtime_binding_instance (instance_id),
UNIQUE KEY uk_instance_runtime_binding_gateway (runtime_pod_id, gateway_port),
KEY idx_instance_runtime_binding_pod_state (runtime_pod_id, state),
CONSTRAINT fk_instance_runtime_binding_instance
FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE,
CONSTRAINT fk_instance_runtime_binding_pod
FOREIGN KEY (runtime_pod_id) REFERENCES runtime_pods(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS runtime_rollouts (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
runtime_type VARCHAR(32) NOT NULL,
target_image_ref VARCHAR(512) NOT NULL,
status VARCHAR(32) NOT NULL DEFAULT 'pending',
batch_size INT NOT NULL DEFAULT 1,
max_unavailable INT NOT NULL DEFAULT 1,
started_by INT NULL,
started_at DATETIME NULL,
finished_at DATETIME NULL,
error_message TEXT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
KEY idx_runtime_rollouts_type_status (runtime_type, status)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS workspace_file_audits (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
instance_id INT NOT NULL,
user_id INT NOT NULL,
action VARCHAR(32) NOT NULL,
relative_path VARCHAR(1024) NOT NULL,
bytes BIGINT NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
KEY idx_workspace_file_audits_instance_time (instance_id, created_at),
KEY idx_workspace_file_audits_user_time (user_id, created_at),
CONSTRAINT fk_workspace_file_audits_instance
FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
backend/internal/db/migrations/024_add_gateway_runtime_type.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE instances
MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop';
backend/internal/db/migrations/025_add_instance_mode.sql
+24
View File
@@ -0,0 +1,24 @@
SET @instance_mode_column_exists = (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'instances'
AND COLUMN_NAME = 'instance_mode'
);
SET @instance_mode_column_sql = IF(
@instance_mode_column_exists = 0,
'ALTER TABLE instances ADD COLUMN instance_mode ENUM(''lite'', ''pro'') NOT NULL DEFAULT ''lite'' AFTER runtime_type',
'SELECT 1'
);
PREPARE instance_mode_column_stmt FROM @instance_mode_column_sql;
EXECUTE instance_mode_column_stmt;
DEALLOCATE PREPARE instance_mode_column_stmt;
UPDATE instances
SET instance_mode = CASE
WHEN runtime_type = 'gateway' THEN 'lite'
ELSE 'pro'
END
WHERE instance_mode IS NULL
OR instance_mode = ''
OR (runtime_type <> 'gateway' AND instance_mode = 'lite');
backend/internal/db/migrations/027_add_instance_external_access.sql
+23
View File
@@ -0,0 +1,23 @@
CREATE TABLE IF NOT EXISTS instance_external_access (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
instance_id INT NOT NULL,
enabled TINYINT(1) NOT NULL DEFAULT 0,
auth_mode VARCHAR(32) NOT NULL DEFAULT 'public_link',
public_slug VARCHAR(64) NULL,
public_token_hash VARCHAR(128) NULL,
short_code_hash VARCHAR(128) NULL,
api_key_hash VARCHAR(128) NULL,
password_value VARCHAR(128) NULL,
api_key_prefix VARCHAR(32) NULL,
expires_at DATETIME NULL,
created_by INT NULL,
last_used_at DATETIME NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_instance_external_access_instance (instance_id),
UNIQUE KEY uk_instance_external_access_slug (public_slug),
UNIQUE KEY uk_instance_external_access_short_code (short_code_hash),
KEY idx_instance_external_access_enabled (enabled, auth_mode),
CONSTRAINT fk_instance_external_access_instance
FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql
+266
View File
@@ -0,0 +1,266 @@
CREATE TABLE IF NOT EXISTS model_invocations (
id INT AUTO_INCREMENT PRIMARY KEY,
trace_id VARCHAR(100) NOT NULL,
session_id VARCHAR(100) NULL,
request_id VARCHAR(100) NOT NULL,
user_id INT NULL,
instance_id INT NULL,
instance_mode VARCHAR(16) NULL,
runtime_type VARCHAR(32) NULL,
gateway_id VARCHAR(128) NULL,
runtime_pod_id BIGINT NULL,
model_id INT NULL,
provider_type VARCHAR(100) NOT NULL,
requested_model VARCHAR(255) NOT NULL,
actual_provider_model VARCHAR(255) NOT NULL,
traffic_class VARCHAR(50) NOT NULL,
request_payload LONGTEXT NULL,
response_payload LONGTEXT NULL,
prompt_tokens INT NOT NULL DEFAULT 0,
completion_tokens INT NOT NULL DEFAULT 0,
total_tokens INT NOT NULL DEFAULT 0,
cached_tokens INT NULL,
reasoning_tokens INT NULL,
latency_ms INT NULL,
is_streaming BOOLEAN NOT NULL DEFAULT FALSE,
status VARCHAR(50) NOT NULL,
error_message TEXT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
completed_at TIMESTAMP NULL,
INDEX idx_model_invocations_trace_id (trace_id),
INDEX idx_model_invocations_session_id (session_id),
INDEX idx_model_invocations_request_id (request_id),
INDEX idx_model_invocations_user_id (user_id),
INDEX idx_model_invocations_instance_id (instance_id),
INDEX idx_model_invocations_gateway_id (gateway_id),
INDEX idx_model_invocations_model_id (model_id),
INDEX idx_model_invocations_status (status),
INDEX idx_model_invocations_created_at (created_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS audit_events (
id INT AUTO_INCREMENT PRIMARY KEY,
trace_id VARCHAR(100) NOT NULL,
session_id VARCHAR(100) NULL,
request_id VARCHAR(100) NULL,
user_id INT NULL,
instance_id INT NULL,
instance_mode VARCHAR(16) NULL,
runtime_type VARCHAR(32) NULL,
gateway_id VARCHAR(128) NULL,
runtime_pod_id BIGINT NULL,
invocation_id INT NULL,
event_type VARCHAR(100) NOT NULL,
traffic_class VARCHAR(50) NOT NULL,
severity VARCHAR(20) NOT NULL,
message VARCHAR(500) NOT NULL,
details LONGTEXT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
INDEX idx_audit_events_trace_id (trace_id),
INDEX idx_audit_events_request_id (request_id),
INDEX idx_audit_events_user_id (user_id),
INDEX idx_audit_events_gateway_id (gateway_id),
INDEX idx_audit_events_invocation_id (invocation_id),
INDEX idx_audit_events_event_type (event_type),
INDEX idx_audit_events_created_at (created_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS cost_records (
id INT AUTO_INCREMENT PRIMARY KEY,
trace_id VARCHAR(100) NOT NULL,
session_id VARCHAR(100) NULL,
request_id VARCHAR(100) NULL,
user_id INT NULL,
instance_id INT NULL,
instance_mode VARCHAR(16) NULL,
runtime_type VARCHAR(32) NULL,
gateway_id VARCHAR(128) NULL,
runtime_pod_id BIGINT NULL,
invocation_id INT NULL,
model_id INT NULL,
provider_type VARCHAR(100) NOT NULL,
model_name VARCHAR(255) NOT NULL,
currency VARCHAR(16) NOT NULL DEFAULT 'USD',
prompt_tokens INT NOT NULL DEFAULT 0,
completion_tokens INT NOT NULL DEFAULT 0,
total_tokens INT NOT NULL DEFAULT 0,
input_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0,
output_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0,
estimated_cost DECIMAL(18,8) NOT NULL DEFAULT 0,
actual_cost DECIMAL(18,8) NULL,
internal_cost DECIMAL(18,8) NOT NULL DEFAULT 0,
recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
INDEX idx_cost_records_trace_id (trace_id),
INDEX idx_cost_records_user_id (user_id),
INDEX idx_cost_records_gateway_id (gateway_id),
INDEX idx_cost_records_model_id (model_id),
INDEX idx_cost_records_provider_type (provider_type),
INDEX idx_cost_records_recorded_at (recorded_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS risk_hits (
id INT AUTO_INCREMENT PRIMARY KEY,
trace_id VARCHAR(100) NOT NULL,
session_id VARCHAR(100) NULL,
request_id VARCHAR(100) NULL,
user_id INT NULL,
instance_id INT NULL,
instance_mode VARCHAR(16) NULL,
runtime_type VARCHAR(32) NULL,
gateway_id VARCHAR(128) NULL,
runtime_pod_id BIGINT NULL,
invocation_id INT NULL,
rule_id VARCHAR(100) NOT NULL,
rule_name VARCHAR(255) NOT NULL,
severity VARCHAR(20) NOT NULL,
action VARCHAR(50) NOT NULL,
match_summary VARCHAR(500) NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
INDEX idx_risk_hits_trace_id (trace_id),
INDEX idx_risk_hits_request_id (request_id),
INDEX idx_risk_hits_user_id (user_id),
INDEX idx_risk_hits_gateway_id (gateway_id),
INDEX idx_risk_hits_invocation_id (invocation_id),
INDEX idx_risk_hits_severity (severity),
INDEX idx_risk_hits_created_at (created_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'instance_mode'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'runtime_type'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'gateway_id'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'runtime_pod_id'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'instance_mode'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'runtime_type'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'gateway_id'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'runtime_pod_id'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'instance_mode'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'runtime_type'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'gateway_id'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'runtime_pod_id'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'instance_mode'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'runtime_type'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'gateway_id'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @column_exists = (
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'runtime_pod_id'
);
SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @index_exists = (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND INDEX_NAME = 'idx_model_invocations_gateway_id'
);
SET @ddl = IF(@index_exists = 0, 'ALTER TABLE model_invocations ADD INDEX idx_model_invocations_gateway_id (gateway_id)', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @index_exists = (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND INDEX_NAME = 'idx_audit_events_gateway_id'
);
SET @ddl = IF(@index_exists = 0, 'ALTER TABLE audit_events ADD INDEX idx_audit_events_gateway_id (gateway_id)', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @index_exists = (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND INDEX_NAME = 'idx_cost_records_gateway_id'
);
SET @ddl = IF(@index_exists = 0, 'ALTER TABLE cost_records ADD INDEX idx_cost_records_gateway_id (gateway_id)', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @index_exists = (
SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND INDEX_NAME = 'idx_risk_hits_gateway_id'
);
SET @ddl = IF(@index_exists = 0, 'ALTER TABLE risk_hits ADD INDEX idx_risk_hits_gateway_id (gateway_id)', 'SELECT 1');
PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
backend/internal/db/migrations/029_add_short_external_access_links.sql
+47
View File
@@ -0,0 +1,47 @@
SET @column_exists = (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'instance_external_access'
AND COLUMN_NAME = 'short_code_hash'
);
SET @ddl = IF(
@column_exists = 0,
'ALTER TABLE instance_external_access ADD COLUMN short_code_hash VARCHAR(128) NULL AFTER public_token_hash',
'SELECT 1'
);
PREPARE stmt FROM @ddl;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @public_slug_exists = (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'instance_external_access'
AND COLUMN_NAME = 'public_slug'
);
SET @ddl = IF(
@public_slug_exists > 0,
'ALTER TABLE instance_external_access MODIFY COLUMN public_slug VARCHAR(64) NULL',
'SELECT 1'
);
PREPARE stmt FROM @ddl;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @index_exists = (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'instance_external_access'
AND INDEX_NAME = 'uk_instance_external_access_short_code'
);
SET @ddl = IF(
@index_exists = 0,
'ALTER TABLE instance_external_access ADD UNIQUE KEY uk_instance_external_access_short_code (short_code_hash)',
'SELECT 1'
);
PREPARE stmt FROM @ddl;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
backend/internal/db/migrations/030_add_external_access_password_value.sql
+15
View File
@@ -0,0 +1,15 @@
SET @column_exists = (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'instance_external_access'
AND COLUMN_NAME = 'password_value'
);
SET @ddl = IF(
@column_exists = 0,
'ALTER TABLE instance_external_access ADD COLUMN password_value VARCHAR(128) NULL AFTER api_key_hash',
'SELECT 1'
);
PREPARE stmt FROM @ddl;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql
+31
View File
@@ -0,0 +1,31 @@
UPDATE system_image_settings
SET
image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
display_name = 'Hermes Lite',
is_enabled = TRUE
WHERE instance_type = 'hermes'
AND runtime_type = 'shell'
AND image IN (
'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest',
'registry.example.com/your-custom-shell-image:latest'
);
INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
SELECT
'hermes',
'shell',
'Hermes Lite',
'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
FROM system_image_settings
WHERE instance_type = 'hermes'
AND runtime_type = 'shell'
)
AND NOT EXISTS (
SELECT 1
FROM system_image_settings
WHERE instance_type = 'hermes'
AND is_enabled = FALSE
);
backend/internal/db/migrations/032_add_team_member_instance_mode.sql
+27
View File
@@ -0,0 +1,27 @@
SET @team_member_instance_mode_column_exists = (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'team_members'
AND COLUMN_NAME = 'instance_mode'
);
SET @team_member_instance_mode_column_sql = IF(
@team_member_instance_mode_column_exists = 0,
'ALTER TABLE team_members ADD COLUMN instance_mode ENUM(''lite'', ''pro'') NOT NULL DEFAULT ''lite'' AFTER runtime_type',
'SELECT 1'
);
PREPARE team_member_instance_mode_column_stmt FROM @team_member_instance_mode_column_sql;
EXECUTE team_member_instance_mode_column_stmt;
DEALLOCATE PREPARE team_member_instance_mode_column_stmt;
UPDATE team_members tm
LEFT JOIN instances i ON i.id = tm.instance_id
SET tm.instance_mode = CASE
WHEN i.instance_mode IN ('lite', 'pro') THEN i.instance_mode
WHEN i.runtime_type = 'gateway' THEN 'lite'
WHEN i.id IS NOT NULL THEN 'pro'
ELSE tm.instance_mode
END
WHERE tm.instance_mode IS NULL
OR tm.instance_mode = ''
OR (i.id IS NOT NULL AND tm.instance_mode <> COALESCE(i.instance_mode, CASE WHEN i.runtime_type = 'gateway' THEN 'lite' ELSE 'pro' END));
backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql
+26
View File
@@ -0,0 +1,26 @@
ALTER TABLE system_image_settings
MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop';
UPDATE system_image_settings
SET runtime_type = 'gateway',
display_name = 'OpenClaw Lite'
WHERE instance_type = 'openclaw'
AND runtime_type = 'shell';
UPDATE system_image_settings
SET runtime_type = 'gateway',
display_name = 'Hermes Lite'
WHERE instance_type = 'hermes'
AND runtime_type = 'shell';
UPDATE system_image_settings
SET display_name = 'OpenClaw Pro'
WHERE instance_type = 'openclaw'
AND runtime_type = 'desktop'
AND display_name IN ('OpenClaw Desktop', 'OpenClaw Runtime');
UPDATE system_image_settings
SET display_name = 'Hermes Pro'
WHERE instance_type = 'hermes'
AND runtime_type = 'desktop'
AND display_name IN ('Hermes Runtime', 'Hermes Desktop');
backend/internal/db/migrations/034_update_lite_default_images.sql
+54
View File
@@ -0,0 +1,54 @@
UPDATE system_image_settings
SET
image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
display_name = 'OpenClaw Lite',
is_enabled = TRUE
WHERE instance_type = 'openclaw'
AND runtime_type IN ('shell', 'gateway')
AND image IN (
'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest',
'172.16.1.12:5010/openclaw-shell:local5',
'172.16.1.12:5010/openclaw:v2dev'
);
UPDATE system_image_settings
SET
image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
display_name = 'Hermes Lite',
is_enabled = TRUE
WHERE instance_type = 'hermes'
AND runtime_type IN ('shell', 'gateway')
AND image IN (
'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest',
'172.16.1.12:5010/hermes:codex-shared-agent-tui-bundle-20260609153019',
'172.16.1.12:5010/hermes:team-lite-tui-dist-20260612174205',
'registry.example.com/your-custom-shell-image:latest'
);
INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
SELECT
'openclaw',
'gateway',
'OpenClaw Lite',
'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
FROM system_image_settings
WHERE instance_type = 'openclaw'
AND runtime_type IN ('shell', 'gateway')
);
INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
SELECT
'hermes',
'gateway',
'Hermes Lite',
'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
FROM system_image_settings
WHERE instance_type = 'hermes'
AND runtime_type IN ('shell', 'gateway')
);
backend/internal/db/migrations_test.go
+67
View File
@@ -2,6 +2,7 @@ package db
import (
"reflect"
"strings"
"testing"
)
@@ -29,3 +30,69 @@ UPDATE demo SET note = "a;quoted" WHERE id = 1;
t.Fatalf("unexpected statements:\nwant: %#v\ngot: %#v", want, got)
}
}
func TestMigration023IsEmbedded(t *testing.T) {
files, err := embeddedMigrations.ReadDir("migrations")
if err != nil {
t.Fatalf("read embedded migrations: %v", err)
}
found := false
for _, file := range files {
if file.Name() == "023_add_runtime_pool_v2.sql" {
found = true
break
}
}
if !found {
t.Fatalf("migration 023_add_runtime_pool_v2.sql is not embedded")
}
}
func TestMigration034UpdatesLiteDefaultImages(t *testing.T) {
raw, err := embeddedMigrations.ReadFile("migrations/034_update_lite_default_images.sql")
if err != nil {
t.Fatalf("read migration 034: %v", err)
}
sql := string(raw)
for _, image := range []string{
"ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest",
"ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest",
} {
if !strings.Contains(sql, image) {
t.Fatalf("migration 034 must update lite image %s", image)
}
}
}
func TestMigration023IsRetrySafe(t *testing.T) {
raw, err := embeddedMigrations.ReadFile("migrations/023_add_runtime_pool_v2.sql")
if err != nil {
t.Fatalf("read migration 023: %v", err)
}
sql := string(raw)
if !strings.Contains(sql, "information_schema.COLUMNS") {
t.Fatalf("migration 023 must guard instance column additions with information_schema.COLUMNS")
}
for _, column := range []string{
"workspace_path",
"workspace_usage_bytes",
"runtime_generation",
"runtime_error_message",
} {
if !strings.Contains(sql, "COLUMN_NAME = '"+column+"'") {
t.Fatalf("migration 023 must guard %s column addition", column)
}
}
for _, table := range []string{
"runtime_pods",
"instance_runtime_bindings",
"runtime_rollouts",
"workspace_file_audits",
} {
if !strings.Contains(sql, "CREATE TABLE IF NOT EXISTS "+table) {
t.Fatalf("migration 023 must create %s idempotently", table)
}
}
}
backend/internal/db/runtime_manifest_test.go
+56
View File
@@ -0,0 +1,56 @@
package db
import (
"os"
"path/filepath"
"regexp"
"strings"
"testing"
)
func TestRuntimeManifestsStartHermesRuntime(t *testing.T) {
repoRoot := filepath.Clean(filepath.Join("..", "..", ".."))
for _, manifest := range []string{
filepath.Join(repoRoot, "deployments", "k8s", "clawmanager.yaml"),
filepath.Join(repoRoot, "deployments", "k3s", "clawmanager.yaml"),
} {
t.Run(manifest, func(t *testing.T) {
raw, err := os.ReadFile(manifest)
if err != nil {
t.Fatalf("read manifest: %v", err)
}
pattern := regexp.MustCompile(`(?s)name:\s+hermes-runtime.*?spec:\s+replicas:\s+([0-9]+)`)
matches := pattern.FindSubmatch(raw)
if len(matches) != 2 {
t.Fatalf("could not find hermes-runtime replicas in %s", manifest)
}
if string(matches[1]) != "1" {
t.Fatalf("expected hermes-runtime replicas 1 in %s, got %s", manifest, matches[1])
}
})
}
}
func TestRuntimeManifestsSeedLiteDefaultImages(t *testing.T) {
repoRoot := filepath.Clean(filepath.Join("..", "..", ".."))
for _, manifest := range []string{
filepath.Join(repoRoot, "deployments", "k8s", "clawmanager.yaml"),
filepath.Join(repoRoot, "deployments", "k3s", "clawmanager.yaml"),
filepath.Join(repoRoot, "backend", "deployments", "k8s", "clawreef-incluster.yaml"),
} {
t.Run(manifest, func(t *testing.T) {
raw, err := os.ReadFile(manifest)
if err != nil {
t.Fatalf("read manifest: %v", err)
}
for _, image := range []string{
"ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest",
"ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest",
} {
if !strings.Contains(string(raw), image) {
t.Fatalf("manifest %s must seed lite image %s", manifest, image)
}
}
})
}
}
backend/internal/handlers/ai_gateway_handler.go
+51
View File
@@ -77,6 +77,13 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) {
}
}
}
if !setStringMetadata(c, &req.InstanceMode, "instanceMode") ||
!setStringMetadata(c, &req.RuntimeType, "runtimeType") ||
!setStringMetadata(c, &req.GatewayID, "gatewayID") ||
!setInt64Metadata(c, &req.RuntimePodID, "runtimePodID") {
utils.Error(c, http.StatusForbidden, "Gateway token metadata does not match request")
return
}
if req.Stream {
traceID, err := h.service.StreamChatCompletions(c.Request.Context(), userID.(int), req, c.Writer)
@@ -107,3 +114,47 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) {
c.Status(response.StatusCode)
_, _ = c.Writer.Write(response.Body)
}
func setStringMetadata(c *gin.Context, field **string, key string) bool {
raw, exists := c.Get(key)
if !exists {
return true
}
value, ok := raw.(string)
if !ok {
return true
}
value = strings.TrimSpace(value)
if value == "" {
return true
}
if *field == nil {
*field = &value
return true
}
return strings.TrimSpace(**field) == value
}
func setInt64Metadata(c *gin.Context, field **int64, key string) bool {
raw, exists := c.Get(key)
if !exists {
return true
}
var value int64
switch typed := raw.(type) {
case int64:
value = typed
case int:
value = int64(typed)
default:
return true
}
if value <= 0 {
return true
}
if *field == nil {
*field = &value
return true
}
return **field == value
}
backend/internal/handlers/instance_handler.go
+535 -32
View File
@@ -3,8 +3,10 @@ package handlers
import (
"errors"
"fmt"
"html"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
@@ -57,10 +59,11 @@ type InstanceHandler struct {
openClawTransferService services.OpenClawTransferService
openClawConfigService services.OpenClawConfigService
skillService services.SkillService
externalAccessService services.InstanceExternalAccessService
}
// NewInstanceHandler creates a new instance handler
func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService) *InstanceHandler {
func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService, externalAccessService services.InstanceExternalAccessService, proxyOptions ...services.InstanceProxyServiceOption) *InstanceHandler {
accessService := services.NewInstanceAccessService()
return &InstanceHandler{
instanceService: instanceService,
@@ -69,11 +72,12 @@ func NewInstanceHandler(instanceService services.InstanceService, instanceAgentS
instanceCommandService: instanceCommandService,
instanceConfigRevisionService: instanceConfigRevisionService,
accessService: accessService,
proxyService: services.NewInstanceProxyService(accessService),
proxyService: services.NewInstanceProxyService(accessService, proxyOptions...),
shellService: services.NewInstanceShellService(),
openClawTransferService: services.NewOpenClawTransferService(),
openClawConfigService: openClawConfigService,
skillService: skillService,
externalAccessService: externalAccessService,
}
}
@@ -98,12 +102,20 @@ type PublishConfigRevisionRequest struct {
SnapshotID int `json:"snapshot_id" binding:"required,min=1"`
}
type ExternalAccessRequest struct {
ExpiresMode string `json:"expires_mode,omitempty"`
ExpiresPreset string `json:"expires_preset,omitempty"`
ExpiresAt *time.Time `json:"expires_at,omitempty"`
}
// CreateInstanceRequest represents a create instance request
type CreateInstanceRequest struct {
Name string `json:"name" binding:"required,min=3,max=50"`
Description *string `json:"description,omitempty"`
Type string `json:"type" binding:"required,oneof=openclaw ubuntu debian centos custom webtop hermes"`
RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop shell"`
Type string `json:"type" binding:"required,oneof=openclaw hermes"`
Mode string `json:"mode" binding:"omitempty,oneof=lite pro"`
InstanceMode string `json:"instance_mode" binding:"omitempty,oneof=lite pro"`
RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=gateway desktop shell"`
CPUCores float64 `json:"cpu_cores" binding:"required,min=0.1,max=32"`
MemoryGB int `json:"memory_gb" binding:"required,min=1,max=128"`
DiskGB int `json:"disk_gb" binding:"required,min=10,max=1000"`
@@ -210,6 +222,8 @@ func (h *InstanceHandler) CreateInstance(c *gin.Context) {
Name: req.Name,
Description: req.Description,
Type: req.Type,
Mode: req.Mode,
InstanceMode: req.InstanceMode,
RuntimeType: req.RuntimeType,
CPUCores: req.CPUCores,
MemoryGB: req.MemoryGB,
@@ -550,6 +564,13 @@ func (h *InstanceHandler) GetInstanceStatus(c *gin.Context) {
return
}
if userRole != "admin" {
utils.Success(c, http.StatusOK, "Instance status retrieved successfully", gin.H{
"instance_status": buildUserSafeInstanceStatus(instance, status),
})
return
}
runtime, _ := h.runtimeStatusService.GetByInstanceID(id)
agent, _ := h.instanceAgentService.GetPayloadByInstanceID(id)
@@ -560,6 +581,43 @@ func (h *InstanceHandler) GetInstanceStatus(c *gin.Context) {
})
}
func buildUserSafeInstanceStatus(instance *models.Instance, status *services.InstanceStatus) map[string]interface{} {
if status == nil {
return map[string]interface{}{}
}
payload := map[string]interface{}{
"instance_id": status.InstanceID,
"status": status.Status,
"availability": status.Availability,
"created_at": status.CreatedAt,
}
if payload["availability"] == "" {
payload["availability"] = availabilityForInstanceStatus(status.Status)
}
if status.StartedAt != nil {
payload["started_at"] = status.StartedAt
}
if instance == nil {
return payload
}
if agentType, ok := services.NormalizeV2RuntimeType(instance.Type); ok && (strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "gateway") || strings.EqualFold(strings.TrimSpace(instance.InstanceMode), "lite")) {
payload["agent_type"] = agentType
payload["workspace_usage_bytes"] = instance.WorkspaceUsageBytes
}
return payload
}
func availabilityForInstanceStatus(status string) string {
switch strings.ToLower(strings.TrimSpace(status)) {
case "running":
return "available"
case "creating":
return "starting"
default:
return "unavailable"
}
}
func (h *InstanceHandler) GetRuntimeDetails(c *gin.Context) {
id, _, ok := h.resolveOwnedInstance(c)
if !ok {
@@ -780,7 +838,7 @@ func (h *InstanceHandler) GenerateAccessToken(c *gin.Context) {
}
// Generate proxy entry URL. The actual Service remains internal-only.
accessURL := h.proxyService.GetProxyURL(instance.ID, "")
accessURL := h.proxyService.GetProxyURLForInstance(instance, "")
if accessURL == "" {
utils.Error(c, http.StatusServiceUnavailable, "Unable to generate access URL")
@@ -818,7 +876,7 @@ func (h *InstanceHandler) GenerateAccessToken(c *gin.Context) {
response := map[string]interface{}{
"token": token.Token,
"access_url": accessURL,
"proxy_url": h.proxyService.GetProxyURL(instance.ID, token.Token),
"proxy_url": h.proxyService.GetProxyURLForInstance(instance, token.Token),
"expires_at": token.ExpiresAt,
}
@@ -930,41 +988,62 @@ func (h *InstanceHandler) ProxyInstance(c *gin.Context) {
return
}
// Get token from query parameter
token := c.Query("token")
if token == "" {
cookieToken, err := c.Cookie(fmt.Sprintf("instance_access_%d", id))
if err != nil || cookieToken == "" {
utils.Error(c, http.StatusBadRequest, "Access token required")
return
}
token = cookieToken
} else {
// Promote the one-time query token into a cookie so iframe subresources and
// websocket requests can reuse it without appending the token everywhere.
c.SetCookie(
fmt.Sprintf("instance_access_%d", id),
token,
int(time.Hour.Seconds()),
fmt.Sprintf("/api/v1/instances/%d/proxy", id),
"",
false,
true,
)
token, ok := h.proxyAccessToken(c, id)
if !ok {
return
}
h.proxyInstanceWithToken(c, id, token)
}
func (h *InstanceHandler) proxyAccessToken(c *gin.Context, id int) (string, bool) {
cookieName := fmt.Sprintf("instance_access_%d", id)
if cookieToken, err := c.Cookie(cookieName); err == nil && strings.TrimSpace(cookieToken) != "" {
if accessToken, validateErr := h.accessService.ValidateToken(cookieToken); validateErr == nil && accessToken.InstanceID == id {
return cookieToken, true
}
}
queryToken := strings.TrimSpace(c.Query("token"))
if queryToken == "" {
utils.Error(c, http.StatusBadRequest, "Access token required")
return "", false
}
accessToken, err := h.accessService.ValidateToken(queryToken)
if err != nil || accessToken.InstanceID != id {
utils.Error(c, http.StatusUnauthorized, "Access token expired or invalid")
return "", false
}
// Promote only a validated ClawManager access token. Runtime applications may
// also use a token query parameter for their own websocket/session protocol.
c.SetCookie(
cookieName,
queryToken,
int(time.Hour.Seconds()),
fmt.Sprintf("/api/v1/instances/%d/proxy", id),
"",
false,
true,
)
return queryToken, true
}
func (h *InstanceHandler) proxyInstanceWithToken(c *gin.Context, id int, token string) {
// Check if it's a WebSocket upgrade request
if strings.EqualFold(c.GetHeader("Upgrade"), "websocket") {
err = h.proxyService.ProxyWebSocket(c.Request.Context(), id, token, c.Writer, c.Request)
if err != nil {
http.Error(c.Writer, err.Error(), http.StatusBadGateway)
if err := h.proxyService.ProxyWebSocket(c.Request.Context(), id, token, c.Writer, c.Request); err != nil {
if errors.Is(err, services.ErrInstanceGatewayUnavailable) {
http.Error(c.Writer, "Instance gateway is not available", http.StatusServiceUnavailable)
} else {
http.Error(c.Writer, err.Error(), http.StatusBadGateway)
}
}
return
}
// Proxy regular HTTP request
err = h.proxyService.ProxyRequest(c.Request.Context(), id, token, c.Writer, c.Request)
if err != nil {
if err := h.proxyService.ProxyRequest(c.Request.Context(), id, token, c.Writer, c.Request); err != nil {
// Log the error
fmt.Printf("Proxy error for instance %d: %v\n", id, err)
@@ -974,6 +1053,8 @@ func (h *InstanceHandler) ProxyInstance(c *gin.Context) {
http.Error(c.Writer, "Access token expired or invalid", http.StatusUnauthorized)
} else if err.Error() == "token does not match instance" {
http.Error(c.Writer, "Token does not match instance", http.StatusForbidden)
} else if errors.Is(err, services.ErrInstanceGatewayUnavailable) {
http.Error(c.Writer, "Instance gateway is not available", http.StatusServiceUnavailable)
} else {
http.Error(c.Writer, fmt.Sprintf("Failed to proxy request: %v", err), http.StatusBadGateway)
}
@@ -1204,6 +1285,428 @@ func (h *InstanceHandler) requireOwnedInstance(c *gin.Context) (*models.Instance
return instance, true
}
func (h *InstanceHandler) GetExternalAccess(c *gin.Context) {
instance, ok := h.requireOwnedInstance(c)
if !ok {
return
}
if h.externalAccessService == nil {
utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
return
}
access, err := h.externalAccessService.Get(c.Request.Context(), instance.ID)
if err != nil {
utils.HandleError(c, err)
return
}
payload := gin.H{"external_access": access}
if shareURL := services.ExternalAccessShareURL(access); shareURL != "" {
payload["share_url"] = shareURL
}
if password := services.ExternalAccessPassword(access); password != "" {
payload["password"] = password
}
utils.Success(c, http.StatusOK, "External access retrieved successfully", payload)
}
func (h *InstanceHandler) EnableShareLink(c *gin.Context) {
instance, ok := h.requireOwnedInstance(c)
if !ok {
return
}
if h.externalAccessService == nil {
utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
return
}
var req ExternalAccessRequest
if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF {
utils.ValidationError(c, err)
return
}
userID, _ := c.Get("userID")
result, err := h.externalAccessService.EnableShareLink(c.Request.Context(), instance.ID, userID.(int), externalAccessExpirationRequest(req))
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Share link enabled successfully", result)
}
func (h *InstanceHandler) CreateExternalAccessPassword(c *gin.Context) {
instance, ok := h.requireOwnedInstance(c)
if !ok {
return
}
if h.externalAccessService == nil {
utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
return
}
var req ExternalAccessRequest
if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF {
utils.ValidationError(c, err)
return
}
userID, _ := c.Get("userID")
result, err := h.externalAccessService.CreatePassword(c.Request.Context(), instance.ID, userID.(int), externalAccessExpirationRequest(req))
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Share link password created successfully", result)
}
func (h *InstanceHandler) DisableExternalAccess(c *gin.Context) {
instance, ok := h.requireOwnedInstance(c)
if !ok {
return
}
if h.externalAccessService == nil {
utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
return
}
if err := h.externalAccessService.Disable(c.Request.Context(), instance.ID); err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "External access disabled successfully", nil)
}
func (h *InstanceHandler) OpenShortExternalAccess(c *gin.Context) {
if h.externalAccessService == nil {
utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
return
}
code := strings.TrimSpace(c.Param("code"))
access, err := h.externalAccessService.ResolveShortLink(c.Request.Context(), code)
if err != nil {
utils.Error(c, http.StatusUnauthorized, err.Error())
return
}
var instance *models.Instance
var token string
canonicalAccessURL := ""
switch access.AuthMode {
case services.ExternalAccessModeShareLink:
if _, err := h.externalAccessService.ValidateShortLink(c.Request.Context(), code, ""); err != nil {
utils.Error(c, http.StatusUnauthorized, err.Error())
return
}
case services.ExternalAccessModePassword:
token = h.validShortLinkAccessToken(c, code, access.InstanceID)
if token == "" {
password := externalPassword(c)
isPasswordFormPost := c.Request.Method == http.MethodPost && password == ""
if isPasswordFormPost {
password = c.PostForm("password")
}
if strings.TrimSpace(password) == "" {
if c.Request.Method == http.MethodGet || c.Request.Method == http.MethodHead {
renderShortLinkPasswordForm(c, code, "")
return
}
utils.Error(c, http.StatusUnauthorized, "share link password is required")
return
}
if _, err := h.externalAccessService.ValidateShortLink(c.Request.Context(), code, password); err != nil {
if c.Request.Method == http.MethodPost || shortExternalAccessWantsHTML(c) {
renderShortLinkPasswordForm(c, code, "Invalid password")
return
}
utils.Error(c, http.StatusUnauthorized, err.Error())
return
}
var ok bool
instance, ok = h.requireExternalAccessInstance(c, access)
if !ok {
return
}
instanceToken, ok := h.issueShortExternalAccessToken(c, instance, code)
if !ok {
return
}
token = instanceToken.Token
canonicalAccessURL = instanceToken.AccessURL
if c.Request.Method == http.MethodPost {
c.Redirect(http.StatusSeeOther, shortExternalAccessEntryPath(code))
return
}
}
default:
utils.Error(c, http.StatusBadRequest, "Unsupported share link mode")
return
}
if instance == nil {
var ok bool
instance, ok = h.requireExternalAccessInstance(c, access)
if !ok {
return
}
}
if token == "" {
instanceToken, ok := h.issueShortExternalAccessToken(c, instance, code)
if !ok {
return
}
token = instanceToken.Token
canonicalAccessURL = instanceToken.AccessURL
} else {
setShortExternalAccessCookies(c, instance.ID, code, token, int(time.Hour.Seconds()))
canonicalAccessURL = h.proxyService.GetProxyURLForInstance(instance, "")
}
originalPath := c.Request.URL.Path
if redirectTarget := shortExternalAccessEntryRedirectTarget(c.Request.Method, originalPath, code, canonicalAccessURL); redirectTarget != "" {
c.Redirect(http.StatusSeeOther, redirectTarget)
return
}
originalRawPath := c.Request.URL.RawPath
originalAuthorization := c.Request.Header.Get("Authorization")
originalPassword := c.Request.Header.Get("X-Password")
c.Request.URL.Path = shortExternalAccessProxyPath(originalPath, code, instance.ID)
c.Request.URL.RawPath = ""
c.Request.Header.Del("Authorization")
c.Request.Header.Del("X-Password")
defer func() {
c.Request.URL.Path = originalPath
c.Request.URL.RawPath = originalRawPath
if originalAuthorization != "" {
c.Request.Header.Set("Authorization", originalAuthorization)
}
if originalPassword != "" {
c.Request.Header.Set("X-Password", originalPassword)
}
}()
h.proxyInstanceWithToken(c, instance.ID, token)
}
func bearerToken(header string) string {
value := strings.TrimSpace(header)
if value == "" {
return ""
}
if strings.HasPrefix(strings.ToLower(value), "bearer ") {
return strings.TrimSpace(value[len("bearer "):])
}
return value
}
func externalPassword(c *gin.Context) string {
if password := strings.TrimSpace(c.GetHeader("X-Password")); password != "" {
return password
}
return bearerToken(c.GetHeader("Authorization"))
}
func externalAccessExpirationRequest(req ExternalAccessRequest) services.ExternalAccessExpirationRequest {
return services.ExternalAccessExpirationRequest{
Mode: strings.TrimSpace(req.ExpiresMode),
Preset: strings.TrimSpace(req.ExpiresPreset),
ExpiresAt: req.ExpiresAt,
}
}
func (h *InstanceHandler) requireExternalAccessInstance(c *gin.Context, access *models.InstanceExternalAccess) (*models.Instance, bool) {
if access == nil {
utils.Error(c, http.StatusUnauthorized, "External access is not enabled")
return nil, false
}
instance, err := h.instanceService.GetByID(access.InstanceID)
if err != nil {
utils.HandleError(c, err)
return nil, false
}
if instance == nil {
utils.Error(c, http.StatusNotFound, "Instance not found")
return nil, false
}
if instance.Status != "running" {
utils.Error(c, http.StatusServiceUnavailable, "Instance is not running")
return nil, false
}
if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "shell") {
utils.Error(c, http.StatusBadRequest, "External desktop access is not available for shell runtime instances")
return nil, false
}
return instance, true
}
func (h *InstanceHandler) issueShortExternalAccessToken(c *gin.Context, instance *models.Instance, code string) (*services.AccessToken, bool) {
accessURL := h.proxyService.GetProxyURLForInstance(instance, "")
if accessURL == "" {
utils.Error(c, http.StatusServiceUnavailable, "Unable to generate access URL")
return nil, false
}
instanceToken, err := h.accessService.GenerateToken(
instance.UserID,
instance.ID,
instance.Type,
accessURL,
h.proxyService.GetTargetPortForInstance(instance),
1*time.Hour,
)
if err != nil {
utils.HandleError(c, err)
return nil, false
}
setShortExternalAccessCookies(c, instance.ID, code, instanceToken.Token, int(time.Hour.Seconds()))
return instanceToken, true
}
func (h *InstanceHandler) validShortLinkAccessToken(c *gin.Context, code string, instanceID int) string {
if h == nil || h.accessService == nil {
return ""
}
token, err := c.Cookie(shortExternalAccessCookieName(code))
if err != nil || strings.TrimSpace(token) == "" {
return ""
}
accessToken, err := h.accessService.ValidateToken(token)
if err != nil || accessToken.InstanceID != instanceID {
return ""
}
return token
}
func setShortExternalAccessCookies(c *gin.Context, instanceID int, code, token string, maxAge int) {
c.SetCookie(
fmt.Sprintf("instance_access_%d", instanceID),
token,
maxAge,
fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID),
"",
false,
true,
)
c.SetCookie(
shortExternalAccessCookieName(code),
token,
maxAge,
shortExternalAccessCookiePath(code),
"",
false,
true,
)
}
func shortExternalAccessCookieName(code string) string {
code = strings.Trim(strings.TrimSpace(code), "/")
var builder strings.Builder
for _, r := range code {
switch {
case r >= 'a' && r <= 'z':
builder.WriteRune(r)
case r >= 'A' && r <= 'Z':
builder.WriteRune(r)
case r >= '0' && r <= '9':
builder.WriteRune(r)
case r == '_' || r == '-':
builder.WriteRune(r)
default:
builder.WriteByte('_')
}
}
if builder.Len() == 0 {
return "share_link_access"
}
return "share_link_access_" + builder.String()
}
func shortExternalAccessCookiePath(code string) string {
code = strings.Trim(strings.TrimSpace(code), "/")
if code == "" {
return "/s"
}
return "/s/" + code
}
func shortExternalAccessEntryPath(code string) string {
return shortExternalAccessCookiePath(code) + "/"
}
func shortExternalAccessEntryRedirectTarget(method, requestPath, code, canonicalPath string) string {
if method != http.MethodGet && method != http.MethodHead {
return ""
}
path := strings.TrimSpace(requestPath)
entryPath := shortExternalAccessEntryPath(code)
if path != entryPath && path != strings.TrimSuffix(entryPath, "/") {
return ""
}
target := strings.TrimSpace(canonicalPath)
if target == "" {
return ""
}
parsed, err := url.Parse(target)
if err != nil || parsed.Path == "" {
return ""
}
if !strings.HasPrefix(parsed.Path, "/api/v1/instances/") {
return ""
}
return parsed.Path
}
func shortExternalAccessWantsHTML(c *gin.Context) bool {
accept := strings.ToLower(c.GetHeader("Accept"))
return strings.Contains(accept, "text/html")
}
func renderShortLinkPasswordForm(c *gin.Context, code, errorMessage string) {
action := html.EscapeString(shortExternalAccessEntryPath(code))
errorHTML := ""
if strings.TrimSpace(errorMessage) != "" {
errorHTML = fmt.Sprintf(`<div class="error">%s</div>`, html.EscapeString(errorMessage))
}
body := fmt.Sprintf(`<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Share link password</title>
<style>
:root { color-scheme: light; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
body { margin: 0; min-height: 100vh; display: grid; place-items: center; background: #f6f8fb; color: #0f172a; }
main { width: min(92vw, 380px); border: 1px solid #e2e8f0; border-radius: 8px; background: #fff; box-shadow: 0 18px 45px rgba(15, 23, 42, .08); padding: 28px; }
h1 { margin: 0 0 18px; font-size: 20px; line-height: 1.25; }
label { display: block; margin-bottom: 8px; font-size: 13px; font-weight: 600; color: #334155; }
input { box-sizing: border-box; width: 100%%; height: 42px; border: 1px solid #cbd5e1; border-radius: 6px; padding: 0 12px; font-size: 14px; outline: none; }
input:focus { border-color: #4f46e5; box-shadow: 0 0 0 3px rgba(79, 70, 229, .14); }
button { width: 100%%; height: 42px; margin-top: 14px; border: 0; border-radius: 6px; background: #4f46e5; color: white; font-size: 14px; font-weight: 700; cursor: pointer; }
.error { margin-bottom: 12px; border: 1px solid #fecaca; border-radius: 6px; background: #fef2f2; color: #b91c1c; padding: 10px 12px; font-size: 13px; }
</style>
</head>
<body>
<main>
<h1>Share link password</h1>
%s
<form method="post" action="%s" autocomplete="off">
<label for="share-link-password">Password</label>
<input id="share-link-password" name="password" type="password" autofocus required>
<button type="submit">Open</button>
</form>
</main>
</body>
</html>`, errorHTML, action)
c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(body))
}
func shortExternalAccessProxyPath(requestPath, code string, instanceID int) string {
internalPrefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
shortPrefix := fmt.Sprintf("/s/%s", strings.Trim(strings.TrimSpace(code), "/"))
path := strings.TrimSpace(requestPath)
if path == "" || path == shortPrefix || path == shortPrefix+"/" {
return internalPrefix + "/"
}
if strings.HasPrefix(path, shortPrefix+"/") {
return internalPrefix + strings.TrimPrefix(path, shortPrefix)
}
return internalPrefix + "/"
}
func sanitizeDownloadName(name string, fallback ...string) string {
name = strings.TrimSpace(name)
if name == "" {
backend/internal/handlers/instance_handler_test.go
+207 -1
View File
@@ -1,6 +1,17 @@
package handlers
import "testing"
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"clawreef/internal/models"
"clawreef/internal/services"
"github.com/gin-gonic/gin"
)
func TestWorkspaceArchiveMaxMiB(t *testing.T) {
t.Setenv(workspaceArchiveMaxMiBEnv, "")
@@ -23,3 +34,198 @@ func TestWorkspaceArchiveMaxMiB(t *testing.T) {
t.Fatalf("expected unparsable archive limit to fall back to %d MiB, got %d", defaultWorkspaceArchiveMaxMiB, got)
}
}
func TestBuildUserSafeInstanceStatusHidesRuntimeSchedulingDetails(t *testing.T) {
podName := "runtime-openclaw-abc"
podNamespace := "clawmanager-system"
podIP := "10.42.0.12"
startedAt := time.Now().UTC()
workspacePath := "/workspaces/openclaw/user-45/instance-123"
instance := &models.Instance{
ID: 123,
Type: "openclaw",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
WorkspaceUsageBytes: 123456,
}
status := &services.InstanceStatus{
InstanceID: 123,
Status: "running",
PodName: &podName,
PodNamespace: &podNamespace,
PodIP: &podIP,
PodStatus: "Running",
StartedAt: &startedAt,
}
payload := buildUserSafeInstanceStatus(instance, status)
for _, forbiddenKey := range []string{"pod_name", "pod_namespace", "pod_ip", "pod_status", "gateway_port", "capacity", "node_name"} {
if _, exists := payload[forbiddenKey]; exists {
t.Fatalf("user-safe status exposed %q: %#v", forbiddenKey, payload)
}
}
if payload["availability"] != "available" {
t.Fatalf("availability = %v, want available", payload["availability"])
}
if payload["agent_type"] != "openclaw" {
t.Fatalf("agent_type = %v, want openclaw", payload["agent_type"])
}
if payload["workspace_usage_bytes"] != int64(123456) {
t.Fatalf("workspace_usage_bytes = %v, want 123456", payload["workspace_usage_bytes"])
}
}
func TestShortExternalAccessProxyPath(t *testing.T) {
tests := []struct {
name string
requestPath string
want string
}{
{name: "entry without slash", requestPath: "/s/abc123", want: "/api/v1/instances/71/proxy/"},
{name: "entry with slash", requestPath: "/s/abc123/", want: "/api/v1/instances/71/proxy/"},
{name: "asset", requestPath: "/s/abc123/assets/index.js", want: "/api/v1/instances/71/proxy/assets/index.js"},
{name: "nested", requestPath: "/s/abc123/apps/openclaw/settings", want: "/api/v1/instances/71/proxy/apps/openclaw/settings"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := shortExternalAccessProxyPath(tt.requestPath, "abc123", 71); got != tt.want {
t.Fatalf("shortExternalAccessProxyPath(%q) = %q, want %q", tt.requestPath, got, tt.want)
}
})
}
}
func TestShortExternalAccessEntryRedirectTarget(t *testing.T) {
tests := []struct {
name string
method string
requestPath string
code string
canonicalPath string
want string
}{
{
name: "html entry redirects to canonical proxy path",
method: http.MethodGet,
requestPath: "/s/sl_abc123/",
code: "sl_abc123",
canonicalPath: "/api/v1/instances/71/proxy/chat/",
want: "/api/v1/instances/71/proxy/chat/",
},
{
name: "entry without trailing slash redirects",
method: http.MethodGet,
requestPath: "/s/sl_abc123",
code: "sl_abc123",
canonicalPath: "/api/v1/instances/71/proxy/chat/",
want: "/api/v1/instances/71/proxy/chat/",
},
{
name: "asset path keeps short proxy handling",
method: http.MethodGet,
requestPath: "/s/sl_abc123/assets/index.js",
code: "sl_abc123",
canonicalPath: "/api/v1/instances/71/proxy/chat/",
want: "",
},
{
name: "post keeps password form handling",
method: http.MethodPost,
requestPath: "/s/sl_abc123/",
code: "sl_abc123",
canonicalPath: "/api/v1/instances/71/proxy/chat/",
want: "",
},
{
name: "target strips accidental token query",
method: http.MethodGet,
requestPath: "/s/sl_abc123/",
code: "sl_abc123",
canonicalPath: "/api/v1/instances/71/proxy/chat/?token=secret",
want: "/api/v1/instances/71/proxy/chat/",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := shortExternalAccessEntryRedirectTarget(tt.method, tt.requestPath, tt.code, tt.canonicalPath); got != tt.want {
t.Fatalf("shortExternalAccessEntryRedirectTarget() = %q, want %q", got, tt.want)
}
})
}
}
func TestRenderShortLinkPasswordForm(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/s/sl_abc123/", nil)
renderShortLinkPasswordForm(c, "sl_abc123", "")
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusOK)
}
body := recorder.Body.String()
for _, want := range []string{
"Share link password",
`name="password"`,
`type="password"`,
`action="/s/sl_abc123/"`,
} {
if !strings.Contains(body, want) {
t.Fatalf("password form missing %q in body:\n%s", want, body)
}
}
if contentType := recorder.Header().Get("Content-Type"); !strings.Contains(contentType, "text/html") {
t.Fatalf("content type = %q, want text/html", contentType)
}
}
func TestProxyAccessTokenPrefersCookieOverRuntimeQueryToken(t *testing.T) {
gin.SetMode(gin.TestMode)
accessService := services.NewInstanceAccessService()
defer accessService.Stop()
token, err := accessService.GenerateToken(1, 76, "hermes", "/api/v1/instances/76/proxy/chat/", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/76/proxy/api/ws?token=hermes-session-token", nil)
c.Request.AddCookie(&http.Cookie{Name: "instance_access_76", Value: token.Token})
handler := &InstanceHandler{accessService: accessService}
got, ok := handler.proxyAccessToken(c, 76)
if !ok {
t.Fatal("proxyAccessToken rejected valid cookie")
}
if got != token.Token {
t.Fatalf("proxyAccessToken = %q, want cookie token", got)
}
if recorder.Code != http.StatusOK {
t.Fatalf("unexpected response status %d", recorder.Code)
}
}
func TestProxyAccessTokenRejectsRuntimeQueryTokenWithoutCookie(t *testing.T) {
gin.SetMode(gin.TestMode)
accessService := services.NewInstanceAccessService()
defer accessService.Stop()
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/76/proxy/api/ws?token=hermes-session-token", nil)
handler := &InstanceHandler{accessService: accessService}
if got, ok := handler.proxyAccessToken(c, 76); ok || got != "" {
t.Fatalf("proxyAccessToken = %q/%v, want rejected", got, ok)
}
if recorder.Code != http.StatusUnauthorized {
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusUnauthorized)
}
}
backend/internal/handlers/runtime_agent_handler.go
+373
View File
@@ -0,0 +1,373 @@
package handlers
import (
"context"
"encoding/json"
"net/http"
"strings"
"time"
"clawreef/internal/config"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services"
"clawreef/internal/utils"
"github.com/gin-gonic/gin"
)
const runtimeAgentTokenHeader = "X-ClawManager-Agent-Token"
type runtimeEventPublisher interface {
Publish(ctx context.Context, eventType string, payload any) error
}
type RuntimeAgentHandler struct {
cfg config.RuntimePoolConfig
podRepo repository.RuntimePodRepository
bindingRepo repository.InstanceRuntimeBindingRepository
events runtimeEventPublisher
}
type runtimeAgentPodIdentity struct {
PodID int64 `json:"pod_id"`
Namespace string `json:"namespace"`
PodName string `json:"pod_name"`
}
type runtimeAgentRegisterRequest struct {
RuntimeType string `json:"runtime_type" binding:"required"`
Namespace string `json:"namespace" binding:"required"`
PodName string `json:"pod_name" binding:"required"`
PodUID *string `json:"pod_uid,omitempty"`
PodIP *string `json:"pod_ip,omitempty"`
NodeName *string `json:"node_name,omitempty"`
DeploymentName string `json:"deployment_name" binding:"required"`
ImageRef string `json:"image_ref" binding:"required"`
AgentEndpoint *string `json:"agent_endpoint,omitempty"`
State string `json:"state"`
Capacity int `json:"capacity"`
UsedSlots int `json:"used_slots"`
Draining bool `json:"draining"`
Metrics json.RawMessage `json:"metrics,omitempty"`
ReportedAt *time.Time `json:"reported_at,omitempty"`
}
type runtimeAgentHeartbeatRequest struct {
runtimeAgentPodIdentity
State string `json:"state" binding:"required"`
UsedSlots int `json:"used_slots"`
Draining bool `json:"draining"`
ReportedAt *time.Time `json:"reported_at,omitempty"`
}
type runtimeAgentMetricsRequest struct {
runtimeAgentPodIdentity
CPUMillisUsed int64 `json:"cpu_millis_used"`
MemoryBytesUsed int64 `json:"memory_bytes_used"`
DiskBytesUsed int64 `json:"disk_bytes_used"`
NetworkRXBytes int64 `json:"network_rx_bytes"`
NetworkTXBytes int64 `json:"network_tx_bytes"`
Metrics json.RawMessage `json:"metrics,omitempty"`
ReportedAt *time.Time `json:"reported_at,omitempty"`
}
type runtimeAgentGatewaysRequest struct {
runtimeAgentPodIdentity
Gateways []runtimeAgentGatewayReport `json:"gateways" binding:"required"`
}
type runtimeAgentGatewayReport struct {
InstanceID int `json:"instance_id" binding:"required"`
GatewayID string `json:"gateway_id"`
GatewayPort int `json:"gateway_port"`
GatewayPID *int `json:"gateway_pid,omitempty"`
State string `json:"state" binding:"required"`
Generation int `json:"generation" binding:"required"`
ErrorMessage *string `json:"error_message,omitempty"`
HealthAt *time.Time `json:"health_at,omitempty"`
}
func NewRuntimeAgentHandler(cfg config.RuntimePoolConfig, podRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, events runtimeEventPublisher) *RuntimeAgentHandler {
return &RuntimeAgentHandler{
cfg: cfg,
podRepo: podRepo,
bindingRepo: bindingRepo,
events: events,
}
}
func (h *RuntimeAgentHandler) Register(c *gin.Context) {
if !h.requireAgentToken(c) {
return
}
var req runtimeAgentRegisterRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
runtimeType, ok := services.NormalizeV2RuntimeType(req.RuntimeType)
if !ok {
utils.Error(c, http.StatusBadRequest, "unsupported runtime type")
return
}
state := strings.TrimSpace(req.State)
if state == "" {
state = "ready"
}
capacity := runtimePodCapacityFromReport(req.Capacity, h.cfg.MaxGatewaysPerPod)
lastSeen := time.Now().UTC()
if req.ReportedAt != nil && !req.ReportedAt.IsZero() {
lastSeen = req.ReportedAt.UTC()
}
var metricsJSON *string
if len(req.Metrics) > 0 {
if !json.Valid(req.Metrics) {
utils.Error(c, http.StatusBadRequest, "metrics must be valid JSON")
return
}
raw := string(req.Metrics)
metricsJSON = &raw
}
pod := &models.RuntimePod{
RuntimeType: runtimeType,
Namespace: strings.TrimSpace(req.Namespace),
PodName: strings.TrimSpace(req.PodName),
PodUID: trimStringPtr(req.PodUID),
PodIP: trimStringPtr(req.PodIP),
NodeName: trimStringPtr(req.NodeName),
DeploymentName: strings.TrimSpace(req.DeploymentName),
ImageRef: strings.TrimSpace(req.ImageRef),
AgentEndpoint: trimStringPtr(req.AgentEndpoint),
State: state,
Capacity: capacity,
UsedSlots: req.UsedSlots,
Draining: req.Draining,
MetricsJSON: metricsJSON,
LastSeenAt: &lastSeen,
CPUMillisUsed: 0,
MemoryBytesUsed: 0,
DiskBytesUsed: 0,
NetworkRXBytes: 0,
NetworkTXBytes: 0,
}
if err := h.podRepo.UpsertFromAgent(c.Request.Context(), pod); err != nil {
utils.HandleError(c, err)
return
}
h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{
"pod_id": pod.ID,
"runtime_type": runtimeType,
"namespace": pod.Namespace,
"pod_name": pod.PodName,
"state": state,
"used_slots": req.UsedSlots,
"capacity": capacity,
"draining": req.Draining,
"last_seen_at": lastSeen,
})
utils.Success(c, http.StatusOK, "Runtime pod registered successfully", gin.H{"pod": pod})
}
func (h *RuntimeAgentHandler) Heartbeat(c *gin.Context) {
if !h.requireAgentToken(c) {
return
}
var req runtimeAgentHeartbeatRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity)
if !ok {
return
}
lastSeen := time.Now().UTC()
if req.ReportedAt != nil && !req.ReportedAt.IsZero() {
lastSeen = req.ReportedAt.UTC()
}
capacity := runtimePodCapacityFromReport(0, h.cfg.MaxGatewaysPerPod)
if err := h.podRepo.UpdateHeartbeat(c.Request.Context(), podID, strings.TrimSpace(req.State), req.UsedSlots, capacity, req.Draining, lastSeen); err != nil {
utils.HandleError(c, err)
return
}
h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{
"pod_id": podID,
"state": strings.TrimSpace(req.State),
"used_slots": req.UsedSlots,
"capacity": capacity,
"draining": req.Draining,
"last_seen_at": lastSeen,
})
utils.Success(c, http.StatusOK, "Runtime pod heartbeat accepted", nil)
}
func (h *RuntimeAgentHandler) ReportMetrics(c *gin.Context) {
if !h.requireAgentToken(c) {
return
}
var req runtimeAgentMetricsRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity)
if !ok {
return
}
var metricsJSON *string
if len(req.Metrics) > 0 {
if !json.Valid(req.Metrics) {
utils.Error(c, http.StatusBadRequest, "metrics must be valid JSON")
return
}
raw := string(req.Metrics)
metricsJSON = &raw
}
lastSeen := time.Now().UTC()
if req.ReportedAt != nil && !req.ReportedAt.IsZero() {
lastSeen = req.ReportedAt.UTC()
}
update := repository.RuntimePodMetricsUpdate{
CPUMillisUsed: req.CPUMillisUsed,
MemoryBytesUsed: req.MemoryBytesUsed,
DiskBytesUsed: req.DiskBytesUsed,
NetworkRXBytes: req.NetworkRXBytes,
NetworkTXBytes: req.NetworkTXBytes,
MetricsJSON: metricsJSON,
LastSeenAt: &lastSeen,
}
if err := h.podRepo.UpdateMetrics(c.Request.Context(), podID, update); err != nil {
utils.HandleError(c, err)
return
}
payload := map[string]any{
"pod_id": podID,
"cpu_millis_used": req.CPUMillisUsed,
"memory_bytes_used": req.MemoryBytesUsed,
"disk_bytes_used": req.DiskBytesUsed,
"network_rx_bytes": req.NetworkRXBytes,
"network_tx_bytes": req.NetworkTXBytes,
"last_seen_at": lastSeen,
}
if metricsJSON != nil {
payload["metrics"] = json.RawMessage(*metricsJSON)
}
h.publish(c.Request.Context(), "runtime_pod_metrics", payload)
utils.Success(c, http.StatusOK, "Runtime pod metrics accepted", nil)
}
func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) {
if !h.requireAgentToken(c) {
return
}
var req runtimeAgentGatewaysRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity)
if !ok {
return
}
for _, gateway := range req.Gateways {
binding, err := h.bindingRepo.GetByInstanceID(c.Request.Context(), gateway.InstanceID)
if err != nil {
utils.HandleError(c, err)
return
}
if binding == nil || binding.RuntimePodID != podID || binding.Generation != gateway.Generation {
continue
}
state := strings.TrimSpace(gateway.State)
switch state {
case "running", "healthy":
if err := h.bindingRepo.UpdateRunning(c.Request.Context(), gateway.InstanceID, gateway.Generation, strings.TrimSpace(gateway.GatewayID), gateway.GatewayPort, gateway.GatewayPID); err != nil {
utils.HandleError(c, err)
return
}
default:
if err := h.bindingRepo.UpdateState(c.Request.Context(), gateway.InstanceID, gateway.Generation, state, gateway.ErrorMessage); err != nil {
utils.HandleError(c, err)
return
}
}
}
h.publish(c.Request.Context(), "runtime_pod_gateways_reported", map[string]any{
"pod_id": podID,
"gateway_count": len(req.Gateways),
})
utils.Success(c, http.StatusOK, "Runtime gateway report accepted", nil)
}
func (h *RuntimeAgentHandler) ReportSkills(c *gin.Context) {
if !h.requireAgentToken(c) {
return
}
var payload map[string]any
if err := c.ShouldBindJSON(&payload); err != nil {
utils.ValidationError(c, err)
return
}
h.publish(c.Request.Context(), "runtime_agent_skills_reported", payload)
utils.Success(c, http.StatusOK, "Runtime agent skills report accepted", nil)
}
func (h *RuntimeAgentHandler) requireAgentToken(c *gin.Context) bool {
if strings.TrimSpace(h.cfg.AgentReportToken) == "" || c.GetHeader(runtimeAgentTokenHeader) != h.cfg.AgentReportToken {
utils.Error(c, http.StatusUnauthorized, "invalid runtime agent token")
return false
}
return true
}
func (h *RuntimeAgentHandler) resolvePodID(c *gin.Context, identity runtimeAgentPodIdentity) (int64, bool) {
if identity.PodID > 0 {
return identity.PodID, true
}
namespace := strings.TrimSpace(identity.Namespace)
podName := strings.TrimSpace(identity.PodName)
if namespace == "" || podName == "" {
utils.Error(c, http.StatusBadRequest, "pod_id or namespace and pod_name are required")
return 0, false
}
pod, err := h.podRepo.GetByNamespaceName(c.Request.Context(), namespace, podName)
if err != nil {
utils.HandleError(c, err)
return 0, false
}
if pod == nil {
utils.Error(c, http.StatusNotFound, "runtime pod not found")
return 0, false
}
return pod.ID, true
}
func (h *RuntimeAgentHandler) publish(ctx context.Context, eventType string, payload any) {
if h.events == nil {
return
}
_ = h.events.Publish(ctx, eventType, payload)
}
func runtimePodCapacityFromReport(reported, configured int) int {
if configured > 0 {
return configured
}
capacity := reported
if capacity <= 0 {
capacity = services.RuntimePodCapacity
}
return capacity
}
func trimStringPtr(value *string) *string {
if value == nil {
return nil
}
trimmed := strings.TrimSpace(*value)
if trimmed == "" {
return nil
}
return &trimmed
}
backend/internal/handlers/runtime_agent_handler_test.go
+342
View File
@@ -0,0 +1,342 @@
package handlers
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"clawreef/internal/config"
"clawreef/internal/models"
"clawreef/internal/repository"
"github.com/gin-gonic/gin"
)
func TestRuntimeAgentHandlerRejectsInvalidToken(t *testing.T) {
gin.SetMode(gin.TestMode)
podRepo := &runtimeAgentHandlerPodRepo{}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, &runtimeAgentHandlerEvents{})
router := gin.New()
router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/metrics/report", bytes.NewBufferString(`{"pod_id":9}`))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Fatalf("status = %d, body = %s, want 401", rec.Code, rec.Body.String())
}
if podRepo.updateMetricsCalls != 0 {
t.Fatalf("UpdateMetrics called %d times, want 0", podRepo.updateMetricsCalls)
}
}
func TestRuntimeAgentHandlerRegisterUsesConfiguredCapacity(t *testing.T) {
gin.SetMode(gin.TestMode)
podRepo := &runtimeAgentHandlerPodRepo{}
events := &runtimeAgentHandlerEvents{}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{
AgentReportToken: "secret",
MaxGatewaysPerPod: 33,
}, podRepo, &runtimeAgentHandlerBindingRepo{}, events)
router := gin.New()
router.POST("/api/v1/runtime-agent/register", handler.Register)
body := `{
"runtime_type": "openclaw",
"namespace": "clawmanager-system",
"pod_name": "openclaw-runtime-test",
"deployment_name": "openclaw-runtime",
"image_ref": "registry/openclaw:v1",
"agent_endpoint": "http://10.0.0.1:19090",
"state": "ready",
"capacity": 10,
"used_slots": 7
}`
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/register", bytes.NewBufferString(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-ClawManager-Agent-Token", "secret")
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
pod := podRepo.podsByID[1]
if pod == nil {
t.Fatal("runtime pod was not persisted")
}
if pod.Capacity != 33 {
t.Fatalf("stored capacity = %d, want configured capacity 33", pod.Capacity)
}
if events.lastType != "runtime_pod_state" {
t.Fatalf("event type = %q, want runtime_pod_state", events.lastType)
}
payload, ok := events.lastPayload.(map[string]any)
if !ok {
t.Fatalf("event payload = %#v, want map", events.lastPayload)
}
if payload["capacity"] != 33 {
t.Fatalf("event capacity = %#v, want configured capacity 33", payload["capacity"])
}
}
func TestRuntimeAgentHandlerHeartbeatUsesConfiguredCapacity(t *testing.T) {
gin.SetMode(gin.TestMode)
podRepo := &runtimeAgentHandlerPodRepo{}
events := &runtimeAgentHandlerEvents{}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{
AgentReportToken: "secret",
MaxGatewaysPerPod: 44,
}, podRepo, &runtimeAgentHandlerBindingRepo{}, events)
router := gin.New()
router.POST("/api/v1/runtime-agent/heartbeat", handler.Heartbeat)
body := `{
"pod_id": 9,
"state": "ready",
"used_slots": 8,
"draining": false
}`
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/heartbeat", bytes.NewBufferString(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-ClawManager-Agent-Token", "secret")
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
if podRepo.updatedPodID != 9 {
t.Fatalf("updated pod id = %d, want 9", podRepo.updatedPodID)
}
if podRepo.lastHeartbeatCapacity != 44 {
t.Fatalf("heartbeat capacity = %d, want configured capacity 44", podRepo.lastHeartbeatCapacity)
}
payload, ok := events.lastPayload.(map[string]any)
if !ok {
t.Fatalf("event payload = %#v, want map", events.lastPayload)
}
if payload["capacity"] != 44 {
t.Fatalf("event capacity = %#v, want configured capacity 44", payload["capacity"])
}
}
func TestRuntimeAgentHandlerMetricsReportUpdatesPodAndPublishesEvent(t *testing.T) {
gin.SetMode(gin.TestMode)
podRepo := &runtimeAgentHandlerPodRepo{}
events := &runtimeAgentHandlerEvents{}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, events)
router := gin.New()
router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics)
body := `{
"pod_id": 9,
"cpu_millis_used": 2400,
"memory_bytes_used": 4096,
"disk_bytes_used": 8192,
"network_rx_bytes": 12,
"network_tx_bytes": 34,
"metrics": {"load": 0.42}
}`
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/metrics/report", bytes.NewBufferString(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-ClawManager-Agent-Token", "secret")
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
if podRepo.updatedPodID != 9 {
t.Fatalf("updated pod id = %d, want 9", podRepo.updatedPodID)
}
if podRepo.lastMetrics.CPUMillisUsed != 2400 || podRepo.lastMetrics.MemoryBytesUsed != 4096 || podRepo.lastMetrics.DiskBytesUsed != 8192 {
t.Fatalf("metrics = %#v, want cpu/memory/disk values from request", podRepo.lastMetrics)
}
if podRepo.lastMetrics.MetricsJSON == nil || *podRepo.lastMetrics.MetricsJSON == "" {
t.Fatalf("metrics json was not persisted")
}
if events.lastType != "runtime_pod_metrics" {
t.Fatalf("event type = %q, want runtime_pod_metrics", events.lastType)
}
payload, ok := events.lastPayload.(map[string]any)
if !ok {
t.Fatalf("event payload = %#v, want map", events.lastPayload)
}
if payload["pod_id"] != int64(9) {
t.Fatalf("event pod_id = %#v, want 9", payload["pod_id"])
}
}
func TestRuntimeAgentHandlerGatewayReportOnlyUpdatesCurrentPodBinding(t *testing.T) {
gin.SetMode(gin.TestMode)
bindingRepo := &runtimeAgentHandlerBindingRepo{
bindings: map[int]*models.InstanceRuntimeBinding{
10: {InstanceID: 10, RuntimePodID: 9, Generation: 2},
11: {InstanceID: 11, RuntimePodID: 99, Generation: 2},
12: {InstanceID: 12, RuntimePodID: 9, Generation: 3},
},
}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, &runtimeAgentHandlerEvents{})
router := gin.New()
router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
body := `{
"pod_id": 9,
"gateways": [
{"instance_id":10,"gateway_id":"gw-10","gateway_port":20010,"state":"running","generation":2},
{"instance_id":11,"gateway_id":"gw-11","gateway_port":20011,"state":"running","generation":2},
{"instance_id":12,"gateway_id":"gw-12","gateway_port":20012,"state":"running","generation":2}
]
}`
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/gateways/report", bytes.NewBufferString(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-ClawManager-Agent-Token", "secret")
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
if bindingRepo.updateRunningCalls != 1 {
t.Fatalf("UpdateRunning calls = %d, want 1", bindingRepo.updateRunningCalls)
}
}
type runtimeAgentHandlerPodRepo struct {
updatedPodID int64
lastMetrics repository.RuntimePodMetricsUpdate
lastHeartbeatCapacity int
updateMetricsCalls int
podsByID map[int64]*models.RuntimePod
}
func (r *runtimeAgentHandlerPodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error {
if r.podsByID == nil {
r.podsByID = map[int64]*models.RuntimePod{}
}
if pod.ID == 0 {
pod.ID = int64(len(r.podsByID) + 1)
}
cp := *pod
r.podsByID[pod.ID] = &cp
return nil
}
func (r *runtimeAgentHandlerPodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) {
if r.podsByID == nil {
return nil, nil
}
return r.podsByID[id], nil
}
func (r *runtimeAgentHandlerPodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) {
for _, pod := range r.podsByID {
if pod.Namespace == namespace && pod.PodName == podName {
return pod, nil
}
}
return nil, nil
}
func (r *runtimeAgentHandlerPodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
return nil, nil
}
func (r *runtimeAgentHandlerPodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
return nil, nil
}
func (r *runtimeAgentHandlerPodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
return false, nil
}
func (r *runtimeAgentHandlerPodRepo) ReleaseSlot(ctx context.Context, podID int64) error {
return nil
}
func (r *runtimeAgentHandlerPodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error {
return nil
}
func (r *runtimeAgentHandlerPodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error {
r.updatedPodID = podID
r.lastHeartbeatCapacity = capacity
return nil
}
func (r *runtimeAgentHandlerPodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error {
return nil
}
func (r *runtimeAgentHandlerPodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error {
r.updatedPodID = podID
r.lastMetrics = metrics
r.updateMetricsCalls++
return nil
}
type runtimeAgentHandlerBindingRepo struct {
updateRunningCalls int
updateStateCalls int
bindings map[int]*models.InstanceRuntimeBinding
}
func (r *runtimeAgentHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
return nil
}
func (r *runtimeAgentHandlerBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
return r.bindings[instanceID], nil
}
func (r *runtimeAgentHandlerBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
return nil, nil
}
func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
return nil, nil
}
func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) {
return nil, nil
}
func (r *runtimeAgentHandlerBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error {
r.updateRunningCalls++
return nil
}
func (r *runtimeAgentHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
r.updateStateCalls++
return nil
}
func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
return nil
}
func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
return nil
}
type runtimeAgentHandlerEvents struct {
lastType string
lastPayload any
}
func (e *runtimeAgentHandlerEvents) Publish(ctx context.Context, eventType string, payload any) error {
e.lastType = eventType
e.lastPayload = payload
return nil
}
backend/internal/handlers/runtime_pool_handler.go
+270
View File
@@ -0,0 +1,270 @@
package handlers
import (
"context"
"log"
"net/http"
"strconv"
"strings"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services"
"clawreef/internal/utils"
"github.com/gin-gonic/gin"
)
type RuntimePoolHandler struct {
podRepo repository.RuntimePodRepository
bindingRepo repository.InstanceRuntimeBindingRepository
rolloutRepo repository.RuntimeRolloutRepository
scheduler *services.RuntimeScheduler
events runtimeEventPublisher
}
const (
runtimePodListFallbackHeartbeatTimeout = 10 * time.Second
runtimePodListStaleWindowMultiplier = 3
)
type startRuntimeRolloutRequest struct {
RuntimeType string `json:"runtime_type" binding:"required"`
TargetImageRef string `json:"target_image_ref" binding:"required"`
BatchSize int `json:"batch_size"`
MaxUnavailable int `json:"max_unavailable"`
}
type runtimePoolPodListItem struct {
models.RuntimePod
AgentReported bool `json:"agent_reported"`
}
func NewRuntimePoolHandler(
podRepo repository.RuntimePodRepository,
bindingRepo repository.InstanceRuntimeBindingRepository,
rolloutRepo repository.RuntimeRolloutRepository,
scheduler *services.RuntimeScheduler,
events runtimeEventPublisher,
) *RuntimePoolHandler {
return &RuntimePoolHandler{
podRepo: podRepo,
bindingRepo: bindingRepo,
rolloutRepo: rolloutRepo,
scheduler: scheduler,
events: events,
}
}
func (h *RuntimePoolHandler) ListPods(c *gin.Context) {
runtimeType := strings.TrimSpace(c.Query("runtime_type"))
if runtimeType != "" {
normalized, ok := services.NormalizeV2RuntimeType(runtimeType)
if !ok {
utils.Error(c, http.StatusBadRequest, "unsupported runtime type")
return
}
runtimeType = normalized
}
pods, err := h.podRepo.List(c.Request.Context(), runtimeType)
if err != nil {
utils.HandleError(c, err)
return
}
currentPods := filterCurrentRuntimePods(pods, time.Now().UTC(), h.runtimePodListHeartbeatTimeout())
items := runtimePoolPodListItems(currentPods, true)
if h.scheduler != nil {
deploymentPods, err := h.scheduler.RuntimeDeploymentPods(c.Request.Context(), runtimeType)
if err != nil {
log.Printf("runtime pool list deployment pods failed: %v", err)
} else {
items = mergeRuntimePoolDeploymentPods(items, deploymentPods)
}
}
utils.Success(c, http.StatusOK, "Runtime pods retrieved successfully", gin.H{"pods": items})
}
func (h *RuntimePoolHandler) GetPodGateways(c *gin.Context) {
podID, ok := parseRuntimePodID(c)
if !ok {
return
}
bindings, err := h.bindingRepo.ListByRuntimePodID(c.Request.Context(), podID)
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Runtime pod gateways retrieved successfully", gin.H{"gateways": bindings})
}
func (h *RuntimePoolHandler) DrainPod(c *gin.Context) {
podID, ok := parseRuntimePodID(c)
if !ok {
return
}
if h.scheduler != nil {
if err := h.scheduler.DrainPod(c.Request.Context(), podID); err != nil {
utils.HandleError(c, err)
return
}
} else if err := h.podRepo.MarkState(c.Request.Context(), podID, "draining", true); err != nil {
utils.HandleError(c, err)
return
}
h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{
"pod_id": podID,
"state": "draining",
"draining": true,
})
utils.Success(c, http.StatusOK, "Runtime pod drain started", nil)
}
func (h *RuntimePoolHandler) StartRollout(c *gin.Context) {
var req startRuntimeRolloutRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
runtimeType, ok := services.NormalizeV2RuntimeType(req.RuntimeType)
if !ok {
utils.Error(c, http.StatusBadRequest, "unsupported runtime type")
return
}
targetImage := strings.TrimSpace(req.TargetImageRef)
if targetImage == "" {
utils.Error(c, http.StatusBadRequest, "target image ref is required")
return
}
batchSize := req.BatchSize
if batchSize <= 0 {
batchSize = 1
}
maxUnavailable := req.MaxUnavailable
if maxUnavailable <= 0 {
maxUnavailable = 1
}
startedBy := currentUserIDPtr(c)
rollout := &models.RuntimeRollout{
RuntimeType: runtimeType,
TargetImageRef: targetImage,
Status: "pending",
BatchSize: batchSize,
MaxUnavailable: maxUnavailable,
StartedBy: startedBy,
}
if err := h.rolloutRepo.Create(c.Request.Context(), rollout); err != nil {
utils.HandleError(c, err)
return
}
if h.scheduler != nil {
if err := h.scheduler.StartRollout(c.Request.Context(), rollout.ID); err != nil {
utils.HandleError(c, err)
return
}
}
h.publish(c.Request.Context(), "runtime_rollout", map[string]any{
"rollout_id": rollout.ID,
"runtime_type": rollout.RuntimeType,
"target_image_ref": rollout.TargetImageRef,
"status": rollout.Status,
"batch_size": rollout.BatchSize,
"max_unavailable": rollout.MaxUnavailable,
"started_by": rollout.StartedBy,
})
utils.Success(c, http.StatusCreated, "Runtime rollout created successfully", gin.H{"rollout": rollout})
}
func (h *RuntimePoolHandler) publish(ctx context.Context, eventType string, payload any) {
if h.events == nil {
return
}
_ = h.events.Publish(ctx, eventType, payload)
}
func (h *RuntimePoolHandler) runtimePodListHeartbeatTimeout() time.Duration {
if h != nil && h.scheduler != nil {
if timeout := h.scheduler.HeartbeatTimeout(); timeout > 0 {
return timeout
}
}
return runtimePodListFallbackHeartbeatTimeout
}
func filterCurrentRuntimePods(pods []models.RuntimePod, now time.Time, heartbeatTimeout time.Duration) []models.RuntimePod {
if heartbeatTimeout <= 0 {
return pods
}
cutoff := now.UTC().Add(-runtimePodListStaleWindowMultiplier * heartbeatTimeout)
current := pods[:0]
for _, pod := range pods {
if pod.LastSeenAt != nil && pod.LastSeenAt.UTC().Before(cutoff) {
continue
}
current = append(current, pod)
}
return current
}
func runtimePoolPodListItems(pods []models.RuntimePod, agentReported bool) []runtimePoolPodListItem {
items := make([]runtimePoolPodListItem, 0, len(pods))
for _, pod := range pods {
items = append(items, runtimePoolPodListItem{
RuntimePod: pod,
AgentReported: agentReported,
})
}
return items
}
func mergeRuntimePoolDeploymentPods(items []runtimePoolPodListItem, deploymentPods []models.RuntimePod) []runtimePoolPodListItem {
seen := map[string]struct{}{}
for _, item := range items {
seen[runtimePoolPodKey(item.Namespace, item.PodName)] = struct{}{}
}
for _, pod := range deploymentPods {
key := runtimePoolPodKey(pod.Namespace, pod.PodName)
if key == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
items = append(items, runtimePoolPodListItem{
RuntimePod: pod,
AgentReported: false,
})
}
return items
}
func runtimePoolPodKey(namespace, podName string) string {
namespace = strings.TrimSpace(namespace)
podName = strings.TrimSpace(podName)
if namespace == "" || podName == "" {
return ""
}
return namespace + "/" + podName
}
func parseRuntimePodID(c *gin.Context) (int64, bool) {
podID, err := strconv.ParseInt(c.Param("id"), 10, 64)
if err != nil || podID <= 0 {
utils.Error(c, http.StatusBadRequest, "invalid runtime pod ID")
return 0, false
}
return podID, true
}
func currentUserIDPtr(c *gin.Context) *int {
raw, ok := c.Get("userID")
if !ok {
return nil
}
userID, ok := raw.(int)
if !ok {
return nil
}
return &userID
}
backend/internal/handlers/runtime_pool_handler_test.go
+532
View File
@@ -0,0 +1,532 @@
package handlers
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"clawreef/internal/middleware"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services"
"clawreef/internal/services/k8s"
"github.com/gin-gonic/gin"
)
func TestRuntimePoolHandlerRejectsNonAdmin(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{})
router := runtimePoolHandlerRouter(20, "user", handler)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil)
router.ServeHTTP(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, body = %s, want 403", rec.Code, rec.Body.String())
}
}
func TestRuntimePoolHandlerListPodsReturnsMetricsForAdmin(t *testing.T) {
gin.SetMode(gin.TestMode)
now := time.Now().UTC()
podIP := "10.42.0.31"
nodeName := "node-a"
handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{pods: []models.RuntimePod{{
ID: 9,
RuntimeType: "openclaw",
PodName: "openclaw-runtime-abcde",
PodIP: &podIP,
NodeName: &nodeName,
State: "ready",
UsedSlots: 37,
Capacity: 100,
Draining: false,
CPUMillisUsed: 13600,
MemoryBytesUsed: 42949672960,
DiskBytesUsed: 214748364800,
NetworkRXBytes: 9223372,
NetworkTXBytes: 19223372,
LastSeenAt: &now,
}}}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{})
router := runtimePoolHandlerRouter(1, "admin", handler)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil)
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
var resp struct {
Data struct {
Pods []struct {
ID int64 `json:"id"`
RuntimeType string `json:"runtime_type"`
PodName string `json:"pod_name"`
CPUMillisUsed int64 `json:"cpu_millis_used"`
MemoryBytesUsed int64 `json:"memory_bytes_used"`
DiskBytesUsed int64 `json:"disk_bytes_used"`
NetworkRXBytes int64 `json:"network_rx_bytes"`
NetworkTXBytes int64 `json:"network_tx_bytes"`
LastSeenAt string `json:"last_seen_at"`
} `json:"pods"`
} `json:"data"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(resp.Data.Pods) != 1 {
t.Fatalf("pods length = %d, want 1", len(resp.Data.Pods))
}
pod := resp.Data.Pods[0]
if pod.ID != 9 || pod.RuntimeType != "openclaw" || pod.PodName != "openclaw-runtime-abcde" {
t.Fatalf("pod identity = %#v, want runtime pod data", pod)
}
if pod.CPUMillisUsed != 13600 || pod.MemoryBytesUsed != 42949672960 || pod.DiskBytesUsed != 214748364800 || pod.NetworkRXBytes != 9223372 || pod.NetworkTXBytes != 19223372 {
t.Fatalf("pod metrics = %#v, want persisted metrics", pod)
}
}
func TestRuntimePoolHandlerListPodsOmitsStaleRuntimePods(t *testing.T) {
gin.SetMode(gin.TestMode)
recentSeen := time.Now().UTC()
staleSeen := recentSeen.Add(-5 * time.Minute)
handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{pods: []models.RuntimePod{
{ID: 1, RuntimeType: "openclaw", PodName: "openclaw-runtime-live", State: "ready", LastSeenAt: &recentSeen},
{ID: 2, RuntimeType: "openclaw", PodName: "openclaw-runtime-deleted", State: "unhealthy", LastSeenAt: &staleSeen},
{ID: 3, RuntimeType: "hermes", PodName: "hermes-runtime-recent-unhealthy", State: "unhealthy", LastSeenAt: &recentSeen},
}}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{})
router := runtimePoolHandlerRouter(1, "admin", handler)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil)
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
var resp struct {
Data struct {
Pods []struct {
ID int64 `json:"id"`
PodName string `json:"pod_name"`
} `json:"pods"`
} `json:"data"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(resp.Data.Pods) != 2 {
t.Fatalf("pods length = %d, want 2: %#v", len(resp.Data.Pods), resp.Data.Pods)
}
for _, pod := range resp.Data.Pods {
if pod.ID == 2 || pod.PodName == "openclaw-runtime-deleted" {
t.Fatalf("stale deleted pod was returned: %#v", pod)
}
}
}
func TestRuntimePoolHandlerListPodsIncludesUnreportedDeploymentPods(t *testing.T) {
gin.SetMode(gin.TestMode)
deployments := &runtimePoolHandlerDeploymentService{
pods: []k8s.RuntimeDeploymentPod{{
RuntimeType: "openclaw",
Namespace: "clawmanager-system",
DeploymentName: "openclaw-runtime",
PodName: "openclaw-runtime-current",
ImageRef: "registry/openclaw-lite:final2",
State: "ready",
}},
}
scheduler := services.NewRuntimeScheduler(
nil,
&runtimePoolHandlerPodRepo{},
nil,
&runtimePoolHandlerRolloutRepo{},
&runtimePoolHandlerAgentClient{},
services.NewRuntimeEventService(nil),
nil,
deployments,
time.Second,
services.WithRuntimeSchedulerNamespace("clawmanager-system"),
services.WithRuntimeSchedulerMaxGatewaysPerPod(33),
)
handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, scheduler, &runtimePoolHandlerEvents{})
router := runtimePoolHandlerRouter(1, "admin", handler)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods?runtime_type=openclaw", nil)
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
var resp struct {
Data struct {
Pods []struct {
ID int64 `json:"id"`
RuntimeType string `json:"runtime_type"`
PodName string `json:"pod_name"`
ImageRef string `json:"image_ref"`
State string `json:"state"`
Capacity int `json:"capacity"`
AgentReported bool `json:"agent_reported"`
DeploymentName string `json:"deployment_name"`
} `json:"pods"`
} `json:"data"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(resp.Data.Pods) != 1 {
t.Fatalf("pods length = %d, want 1: %#v", len(resp.Data.Pods), resp.Data.Pods)
}
pod := resp.Data.Pods[0]
if pod.ID >= 0 || pod.AgentReported {
t.Fatalf("fallback pod identity = id:%d agent_reported:%t, want synthetic unreported pod", pod.ID, pod.AgentReported)
}
if pod.RuntimeType != "openclaw" || pod.PodName != "openclaw-runtime-current" || pod.DeploymentName != "openclaw-runtime" {
t.Fatalf("fallback pod metadata = %#v, want OpenClaw deployment pod", pod)
}
if pod.ImageRef != "registry/openclaw-lite:final2" {
t.Fatalf("fallback pod image = %q, want registry/openclaw-lite:final2", pod.ImageRef)
}
if pod.State != "unhealthy" {
t.Fatalf("fallback pod state = %q, want unhealthy while agent is not reporting", pod.State)
}
if pod.Capacity != 33 {
t.Fatalf("fallback pod capacity = %d, want configured capacity 33", pod.Capacity)
}
}
func TestRuntimePoolHandlerStartRolloutStoresRequesterAndPublishesEvent(t *testing.T) {
gin.SetMode(gin.TestMode)
rolloutRepo := &runtimePoolHandlerRolloutRepo{}
events := &runtimePoolHandlerEvents{}
handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, rolloutRepo, nil, events)
router := runtimePoolHandlerRouter(7, "admin", handler)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/runtime-rollouts", bytes.NewBufferString(`{
"runtime_type": "hermes",
"target_image_ref": "ghcr.io/example/hermes:v2",
"batch_size": 2,
"max_unavailable": 1
}`))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Fatalf("status = %d, body = %s, want 201", rec.Code, rec.Body.String())
}
if rolloutRepo.created == nil {
t.Fatalf("rollout was not created")
}
if rolloutRepo.created.StartedBy == nil || *rolloutRepo.created.StartedBy != 7 {
t.Fatalf("started_by = %#v, want 7", rolloutRepo.created.StartedBy)
}
if events.lastType != "runtime_rollout" {
t.Fatalf("event type = %q, want runtime_rollout", events.lastType)
}
}
func TestRuntimePoolHandlerStartRolloutRunsSchedulerImmediately(t *testing.T) {
gin.SetMode(gin.TestMode)
rolloutRepo := &runtimePoolHandlerRolloutRepo{}
podRepo := &runtimePoolHandlerPodRepo{pods: []models.RuntimePod{{
ID: 21,
RuntimeType: "hermes",
Namespace: "clawmanager-system",
PodName: "hermes-runtime-old",
DeploymentName: "hermes-runtime",
ImageRef: "registry/hermes:v1",
State: "ready",
Capacity: 100,
UsedSlots: 7,
}}}
deployments := &runtimePoolHandlerDeploymentService{}
scheduler := services.NewRuntimeScheduler(
nil,
podRepo,
nil,
rolloutRepo,
&runtimePoolHandlerAgentClient{},
services.NewRuntimeEventService(nil),
nil,
deployments,
time.Second,
)
handler := NewRuntimePoolHandler(podRepo, &runtimePoolHandlerBindingRepo{}, rolloutRepo, scheduler, &runtimePoolHandlerEvents{})
router := runtimePoolHandlerRouter(7, "admin", handler)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/runtime-rollouts", bytes.NewBufferString(`{
"runtime_type": "hermes",
"target_image_ref": "registry/hermes:v2",
"batch_size": 1,
"max_unavailable": 1
}`))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Fatalf("status = %d, body = %s, want 201", rec.Code, rec.Body.String())
}
if got := len(deployments.rollouts); got != 1 {
t.Fatalf("deployment rollouts = %d, want 1", got)
}
rollout := deployments.rollouts[0]
if rollout.namespace != "clawmanager-system" || rollout.name != "hermes-runtime" || rollout.image != "registry/hermes:v2" {
t.Fatalf("deployment rollout = %+v, want hermes-runtime registry/hermes:v2", rollout)
}
if podRepo.markedPodID != 21 || podRepo.markedState != "draining" || !podRepo.markedDraining {
t.Fatalf("pod mark = id:%d state:%q draining:%t, want drain pod 21", podRepo.markedPodID, podRepo.markedState, podRepo.markedDraining)
}
}
func runtimePoolHandlerRouter(userID int, role string, handler *RuntimePoolHandler) *gin.Engine {
userRepo := &runtimePoolHandlerUserRepo{users: map[int]*models.User{
userID: {ID: userID, Username: "tester", Role: role},
}}
router := gin.New()
admin := router.Group("/api/v1/admin")
admin.Use(func(c *gin.Context) {
c.Set("userID", userID)
c.Next()
})
admin.Use(middleware.SetUserInfo(userRepo))
admin.Use(middleware.NewAdminAuth(userRepo))
{
admin.GET("/runtime-pods", handler.ListPods)
admin.POST("/runtime-rollouts", handler.StartRollout)
}
return router
}
type runtimePoolHandlerUserRepo struct {
users map[int]*models.User
}
func (r *runtimePoolHandlerUserRepo) Create(user *models.User) error { return nil }
func (r *runtimePoolHandlerUserRepo) GetByID(id int) (*models.User, error) {
return r.users[id], nil
}
func (r *runtimePoolHandlerUserRepo) GetByUsername(username string) (*models.User, error) {
return nil, nil
}
func (r *runtimePoolHandlerUserRepo) GetByEmail(email string) (*models.User, error) {
return nil, nil
}
func (r *runtimePoolHandlerUserRepo) Update(user *models.User) error { return nil }
func (r *runtimePoolHandlerUserRepo) Delete(id int) error { return nil }
func (r *runtimePoolHandlerUserRepo) List(offset, limit int) ([]models.User, error) {
return nil, nil
}
func (r *runtimePoolHandlerUserRepo) Count() (int, error) { return 0, nil }
type runtimePoolHandlerPodRepo struct {
pods []models.RuntimePod
markedPodID int64
markedState string
markedDraining bool
}
func (r *runtimePoolHandlerPodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error {
return nil
}
func (r *runtimePoolHandlerPodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) {
for _, pod := range r.pods {
if pod.ID == id {
cp := pod
return &cp, nil
}
}
return nil, nil
}
func (r *runtimePoolHandlerPodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) {
return nil, nil
}
func (r *runtimePoolHandlerPodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
if runtimeType == "" {
return append([]models.RuntimePod(nil), r.pods...), nil
}
var pods []models.RuntimePod
for _, pod := range r.pods {
if pod.RuntimeType == runtimeType {
pods = append(pods, pod)
}
}
return pods, nil
}
func (r *runtimePoolHandlerPodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
return nil, nil
}
func (r *runtimePoolHandlerPodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
return false, nil
}
func (r *runtimePoolHandlerPodRepo) ReleaseSlot(ctx context.Context, podID int64) error {
return nil
}
func (r *runtimePoolHandlerPodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error {
r.markedPodID = podID
r.markedState = state
r.markedDraining = draining
return nil
}
func (r *runtimePoolHandlerPodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error {
return nil
}
func (r *runtimePoolHandlerPodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error {
return nil
}
func (r *runtimePoolHandlerPodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error {
return nil
}
type runtimePoolHandlerBindingRepo struct {
bindings []models.InstanceRuntimeBinding
}
func (r *runtimePoolHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
return nil
}
func (r *runtimePoolHandlerBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
return nil, nil
}
func (r *runtimePoolHandlerBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
return nil, nil
}
func (r *runtimePoolHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
var bindings []models.InstanceRuntimeBinding
for _, binding := range r.bindings {
if binding.RuntimePodID == runtimePodID {
bindings = append(bindings, binding)
}
}
return bindings, nil
}
func (r *runtimePoolHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) {
return nil, nil
}
func (r *runtimePoolHandlerBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error {
return nil
}
func (r *runtimePoolHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
return nil
}
func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
return nil
}
func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
return nil
}
type runtimePoolHandlerRolloutRepo struct {
created *models.RuntimeRollout
byID map[int64]*models.RuntimeRollout
}
func (r *runtimePoolHandlerRolloutRepo) Create(ctx context.Context, rollout *models.RuntimeRollout) error {
rollout.ID = 12
cp := *rollout
r.created = &cp
if r.byID == nil {
r.byID = map[int64]*models.RuntimeRollout{}
}
r.byID[rollout.ID] = &cp
return nil
}
func (r *runtimePoolHandlerRolloutRepo) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) {
if r.byID == nil {
return nil, nil
}
return r.byID[id], nil
}
func (r *runtimePoolHandlerRolloutRepo) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) {
return nil, nil
}
func (r *runtimePoolHandlerRolloutRepo) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error {
return nil
}
type runtimePoolHandlerEvents struct {
lastType string
lastPayload any
}
func (e *runtimePoolHandlerEvents) Publish(ctx context.Context, eventType string, payload any) error {
e.lastType = eventType
e.lastPayload = payload
return nil
}
type runtimePoolHandlerDeploymentService struct {
rollouts []runtimePoolHandlerDeploymentRollout
pods []k8s.RuntimeDeploymentPod
}
type runtimePoolHandlerDeploymentRollout struct {
namespace string
name string
image string
maxUnavailable int
maxSurge int
}
func (s *runtimePoolHandlerDeploymentService) Ensure(ctx context.Context, spec k8s.RuntimeDeploymentSpec) error {
return nil
}
func (s *runtimePoolHandlerDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error {
return nil
}
func (s *runtimePoolHandlerDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error {
s.rollouts = append(s.rollouts, runtimePoolHandlerDeploymentRollout{
namespace: namespace,
name: name,
image: image,
maxUnavailable: maxUnavailable,
maxSurge: maxSurge,
})
return nil
}
func (s *runtimePoolHandlerDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]k8s.RuntimeDeploymentPod, error) {
var pods []k8s.RuntimeDeploymentPod
for _, pod := range s.pods {
if namespace != "" && pod.Namespace != namespace {
continue
}
if runtimeType != "" && pod.RuntimeType != runtimeType {
continue
}
pods = append(pods, pod)
}
return pods, nil
}
type runtimePoolHandlerAgentClient struct{}
func (c *runtimePoolHandlerAgentClient) Health(ctx context.Context, endpoint string) error {
return nil
}
func (c *runtimePoolHandlerAgentClient) CreateGateway(ctx context.Context, endpoint string, req services.RuntimeAgentCreateGatewayRequest) (*services.RuntimeAgentCreateGatewayResponse, error) {
return nil, nil
}
func (c *runtimePoolHandlerAgentClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error {
return nil
}
func (c *runtimePoolHandlerAgentClient) Drain(ctx context.Context, endpoint string) error { return nil }
backend/internal/handlers/system_settings_handler.go
+1 -1
View File
@@ -19,7 +19,7 @@ type SystemSettingsHandler struct {
type UpsertSystemImageSettingRequest struct {
ID int `json:"id,omitempty"`
InstanceType string `json:"instance_type" binding:"required"`
RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop shell"`
RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop gateway shell"`
DisplayName string `json:"display_name"`
Image string `json:"image" binding:"required"`
}
backend/internal/handlers/websocket_handler.go
+11 -1
View File
@@ -27,9 +27,19 @@ func (h *WebSocketHandler) HandleWebSocket(c *gin.Context) {
c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
return
}
roleRaw, _ := c.Get("userRole")
role, _ := roleRaw.(string)
topic := services.WebSocketTopicUser
if c.Query("topic") == string(services.WebSocketTopicRuntimeAdmin) {
if role != "admin" {
c.JSON(http.StatusForbidden, gin.H{"error": "Admin access required"})
return
}
topic = services.WebSocketTopicRuntimeAdmin
}
// Upgrade HTTP connection to WebSocket
services.ServeWS(h.hub, c.Writer, c.Request, userID.(int))
services.ServeWSWithOptions(h.hub, c.Writer, c.Request, userID.(int), role, topic)
}
// GetConnectionCount returns the number of active WebSocket connections
backend/internal/handlers/workspace_file_handler.go
+317
View File
@@ -0,0 +1,317 @@
package handlers
import (
"errors"
"fmt"
"io"
"mime"
"net/http"
"net/url"
"path/filepath"
"strconv"
"strings"
"unicode"
"clawreef/internal/models"
"clawreef/internal/services"
"clawreef/internal/utils"
"github.com/gin-gonic/gin"
)
type WorkspaceFileHandler struct {
instanceService services.InstanceService
fileService services.WorkspaceFileService
runtimeWorkspaceFileService services.WorkspaceFileService
}
type createWorkspaceFolderRequest struct {
Path string `json:"path" binding:"required"`
}
type renameWorkspaceEntryRequest struct {
OldPath string `json:"old_path" binding:"required"`
NewPath string `json:"new_path" binding:"required"`
}
func NewWorkspaceFileHandler(instanceService services.InstanceService, fileService services.WorkspaceFileService, runtimeWorkspaceFileService ...services.WorkspaceFileService) *WorkspaceFileHandler {
runtimeFileService := fileService
if len(runtimeWorkspaceFileService) > 0 && runtimeWorkspaceFileService[0] != nil {
runtimeFileService = runtimeWorkspaceFileService[0]
}
return &WorkspaceFileHandler{
instanceService: instanceService,
fileService: fileService,
runtimeWorkspaceFileService: runtimeFileService,
}
}
func (h *WorkspaceFileHandler) List(c *gin.Context) {
_, service, scope, ok := h.workspaceScope(c)
if !ok {
return
}
entries, err := service.List(c.Request.Context(), scope, c.Query("path"))
if err != nil {
handleWorkspaceFileError(c, err)
return
}
utils.Success(c, http.StatusOK, "Workspace files retrieved successfully", gin.H{"entries": entries})
}
func (h *WorkspaceFileHandler) Preview(c *gin.Context) {
_, service, scope, ok := h.workspaceScope(c)
if !ok {
return
}
if strings.EqualFold(strings.TrimSpace(c.Query("raw")), "1") {
file, contentType, size, err := service.OpenPreview(c.Request.Context(), scope, c.Query("path"))
if err != nil {
handleWorkspaceFileError(c, err)
return
}
defer file.Close()
filename := safeWorkspaceDownloadName(filepath.Base(c.Query("path")))
streamWorkspaceFile(c, file, filename, contentType, "inline", size)
return
}
preview, err := service.Preview(c.Request.Context(), scope, c.Query("path"))
if err != nil {
handleWorkspaceFileError(c, err)
return
}
utils.Success(c, http.StatusOK, "Workspace preview retrieved successfully", gin.H{"preview": preview})
}
func (h *WorkspaceFileHandler) Download(c *gin.Context) {
_, service, scope, ok := h.workspaceScope(c)
if !ok {
return
}
file, filename, size, err := service.OpenDownload(c.Request.Context(), scope, c.Query("path"))
if err != nil {
handleWorkspaceFileError(c, err)
return
}
defer file.Close()
contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
if contentType == "" {
contentType = "application/octet-stream"
}
streamWorkspaceFile(c, file, filename, contentType, "attachment", size)
}
func (h *WorkspaceFileHandler) Upload(c *gin.Context) {
_, service, scope, ok := h.workspaceScope(c)
if !ok {
return
}
maxBytes := services.WorkspaceUploadMaxBytes()
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes+(1<<20))
fileHeader, err := c.FormFile("file")
if err != nil {
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
utils.Error(c, http.StatusRequestEntityTooLarge, "workspace upload exceeds maximum size")
return
}
utils.Error(c, http.StatusBadRequest, "file is required")
return
}
if fileHeader.Size > maxBytes {
utils.Error(c, http.StatusRequestEntityTooLarge, "workspace upload exceeds maximum size")
return
}
file, err := fileHeader.Open()
if err != nil {
handleWorkspaceFileError(c, err)
return
}
defer file.Close()
entry, err := service.Upload(c.Request.Context(), scope, c.Query("path"), fileHeader.Filename, file, fileHeader.Size)
if err != nil {
handleWorkspaceFileError(c, err)
return
}
utils.Success(c, http.StatusCreated, "Workspace file uploaded successfully", entry)
}
func (h *WorkspaceFileHandler) Mkdir(c *gin.Context) {
_, service, scope, ok := h.workspaceScope(c)
if !ok {
return
}
var req createWorkspaceFolderRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
entry, err := service.Mkdir(c.Request.Context(), scope, req.Path)
if err != nil {
handleWorkspaceFileError(c, err)
return
}
utils.Success(c, http.StatusCreated, "Workspace folder created successfully", entry)
}
func (h *WorkspaceFileHandler) Rename(c *gin.Context) {
_, service, scope, ok := h.workspaceScope(c)
if !ok {
return
}
var req renameWorkspaceEntryRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
entry, err := service.Rename(c.Request.Context(), scope, req.OldPath, req.NewPath)
if err != nil {
handleWorkspaceFileError(c, err)
return
}
utils.Success(c, http.StatusOK, "Workspace entry renamed successfully", entry)
}
func (h *WorkspaceFileHandler) Delete(c *gin.Context) {
_, service, scope, ok := h.workspaceScope(c)
if !ok {
return
}
if err := service.Delete(c.Request.Context(), scope, c.Query("path")); err != nil {
handleWorkspaceFileError(c, err)
return
}
utils.Success(c, http.StatusOK, "Workspace entry deleted successfully", nil)
}
func (h *WorkspaceFileHandler) workspaceScope(c *gin.Context) (*models.Instance, services.WorkspaceFileService, services.WorkspaceFileScope, bool) {
id, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "Invalid instance ID")
return nil, nil, services.WorkspaceFileScope{}, false
}
instance, err := h.instanceService.GetByID(id)
if err != nil {
utils.HandleError(c, err)
return nil, nil, services.WorkspaceFileScope{}, false
}
if instance == nil {
utils.Error(c, http.StatusNotFound, "Instance not found")
return nil, nil, services.WorkspaceFileScope{}, false
}
userIDRaw, ok := c.Get("userID")
if !ok {
utils.Error(c, http.StatusUnauthorized, "Unauthorized")
return nil, nil, services.WorkspaceFileScope{}, false
}
userID, ok := userIDRaw.(int)
if !ok {
utils.Error(c, http.StatusUnauthorized, "Unauthorized")
return nil, nil, services.WorkspaceFileScope{}, false
}
roleRaw, _ := c.Get("userRole")
userRole, _ := roleRaw.(string)
if userRole != "admin" && instance.UserID != userID {
utils.Error(c, http.StatusForbidden, "Access denied")
return nil, nil, services.WorkspaceFileScope{}, false
}
if isDesktopWorkspaceInstance(instance) {
return instance, h.runtimeWorkspaceFileService, services.WorkspaceFileScope{
InstanceID: instance.ID,
UserID: instance.UserID,
WorkspacePath: "/config",
}, true
}
if instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" {
utils.Error(c, http.StatusNotFound, "Workspace not found")
return nil, nil, services.WorkspaceFileScope{}, false
}
return instance, h.fileService, services.WorkspaceFileScope{
InstanceID: instance.ID,
UserID: instance.UserID,
WorkspacePath: *instance.WorkspacePath,
}, true
}
func isDesktopWorkspaceInstance(instance *models.Instance) bool {
if instance == nil {
return false
}
return strings.EqualFold(strings.TrimSpace(instance.InstanceMode), services.InstanceModePro) ||
strings.EqualFold(strings.TrimSpace(instance.RuntimeType), services.RuntimeBackendDesktop)
}
func streamWorkspaceFile(c *gin.Context, file io.ReadSeeker, filename, contentType, disposition string, size int64) {
safeName := safeWorkspaceDownloadName(filename)
c.Header("Content-Type", contentType)
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Content-Disposition", workspaceContentDisposition(disposition, safeName))
if size >= 0 {
c.Header("Content-Length", strconv.FormatInt(size, 10))
}
if _, err := io.Copy(c.Writer, file); err != nil && !c.Writer.Written() {
utils.HandleError(c, err)
}
}
func workspaceContentDisposition(disposition, filename string) string {
disposition = strings.TrimSpace(strings.ToLower(disposition))
if disposition != "inline" {
disposition = "attachment"
}
escaped := strings.ReplaceAll(filename, `\`, `\\`)
escaped = strings.ReplaceAll(escaped, `"`, `\"`)
return fmt.Sprintf("%s; filename=\"%s\"; filename*=UTF-8''%s", disposition, escaped, url.PathEscape(filename))
}
func safeWorkspaceDownloadName(name string) string {
name = strings.TrimSpace(strings.ReplaceAll(name, "\\", "/"))
name = filepath.Base(name)
if name == "." || name == "/" || name == "" {
name = "download"
}
name = strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return -1
}
switch r {
case '/', '\\', ':', '*', '?', '"', '<', '>', '|':
return '-'
default:
return r
}
}, name)
name = strings.TrimSpace(name)
if name == "" || name == "." || name == ".." {
return "download"
}
return name
}
func handleWorkspaceFileError(c *gin.Context, err error) {
switch {
case errors.Is(err, services.ErrWorkspacePathNotFound):
utils.Error(c, http.StatusNotFound, err.Error())
case errors.Is(err, services.ErrWorkspacePreviewTooLarge), errors.Is(err, services.ErrWorkspaceUploadTooLarge):
utils.Error(c, http.StatusRequestEntityTooLarge, err.Error())
case errors.Is(err, services.ErrWorkspacePathEscape):
utils.Error(c, http.StatusForbidden, "Access denied")
case errors.Is(err, services.ErrWorkspacePathInvalid),
errors.Is(err, services.ErrWorkspaceDirectoryExpected),
errors.Is(err, services.ErrWorkspaceFileExpected),
errors.Is(err, services.ErrWorkspaceRootOperation),
errors.Is(err, services.ErrWorkspaceFileNameInvalid),
errors.Is(err, services.ErrWorkspaceEntryExists):
utils.Error(c, http.StatusBadRequest, err.Error())
default:
utils.HandleError(c, err)
}
}
backend/internal/handlers/workspace_file_handler_test.go
+242
View File
@@ -0,0 +1,242 @@
package handlers
import (
"context"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
"clawreef/internal/models"
"clawreef/internal/services"
"github.com/gin-gonic/gin"
)
type fakeWorkspaceHandlerInstanceService struct {
instances map[int]*models.Instance
}
func (s *fakeWorkspaceHandlerInstanceService) Create(userID int, req services.CreateInstanceRequest) (*models.Instance, error) {
return nil, nil
}
func (s *fakeWorkspaceHandlerInstanceService) ValidateCreateRequests(userID int, requests []services.CreateInstanceRequest) error {
return nil
}
func (s *fakeWorkspaceHandlerInstanceService) GetByID(id int) (*models.Instance, error) {
return s.instances[id], nil
}
func (s *fakeWorkspaceHandlerInstanceService) GetByUserID(userID int, offset, limit int) ([]models.Instance, int, error) {
return nil, 0, nil
}
func (s *fakeWorkspaceHandlerInstanceService) GetAllInstances(offset, limit int) ([]models.Instance, int, error) {
return nil, 0, nil
}
func (s *fakeWorkspaceHandlerInstanceService) Start(instanceID int) error { return nil }
func (s *fakeWorkspaceHandlerInstanceService) Stop(instanceID int) error { return nil }
func (s *fakeWorkspaceHandlerInstanceService) Restart(instanceID int) error { return nil }
func (s *fakeWorkspaceHandlerInstanceService) Delete(instanceID int) error { return nil }
func (s *fakeWorkspaceHandlerInstanceService) Update(instanceID int, req services.UpdateInstanceRequest) error {
return nil
}
func (s *fakeWorkspaceHandlerInstanceService) GetInstanceStatus(instanceID int) (*services.InstanceStatus, error) {
return nil, nil
}
func (s *fakeWorkspaceHandlerInstanceService) ForceSyncInstance(instanceID int) error { return nil }
type fakeWorkspaceFileService struct {
lastScope services.WorkspaceFileScope
listCalls int
file *os.File
filename string
size int64
}
func (s *fakeWorkspaceFileService) List(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) ([]services.WorkspaceEntry, error) {
s.lastScope = scope
s.listCalls++
return []services.WorkspaceEntry{{Name: "readme.md", Path: "readme.md", ModifiedAt: time.Unix(1, 0), Previewable: true, Downloadable: true}}, nil
}
func (s *fakeWorkspaceFileService) Preview(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*services.WorkspacePreview, error) {
s.lastScope = scope
return &services.WorkspacePreview{Kind: "text", Text: "ok"}, nil
}
func (s *fakeWorkspaceFileService) OpenPreview(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
s.lastScope = scope
return s.file, "text/plain; charset=utf-8", s.size, nil
}
func (s *fakeWorkspaceFileService) OpenDownload(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
s.lastScope = scope
return s.file, s.filename, s.size, nil
}
func (s *fakeWorkspaceFileService) Upload(ctx context.Context, scope services.WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*services.WorkspaceEntry, error) {
s.lastScope = scope
return &services.WorkspaceEntry{Name: filename, Path: filename, Downloadable: true}, nil
}
func (s *fakeWorkspaceFileService) Mkdir(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*services.WorkspaceEntry, error) {
s.lastScope = scope
return &services.WorkspaceEntry{Name: "docs", Path: relativePath, IsDir: true}, nil
}
func (s *fakeWorkspaceFileService) Rename(ctx context.Context, scope services.WorkspaceFileScope, oldPath, newPath string) (*services.WorkspaceEntry, error) {
s.lastScope = scope
return &services.WorkspaceEntry{Name: "renamed", Path: newPath}, nil
}
func (s *fakeWorkspaceFileService) Delete(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) error {
s.lastScope = scope
return nil
}
func TestWorkspaceFileHandlerRejectsNonOwnerUser(t *testing.T) {
gin.SetMode(gin.TestMode)
workspacePath := t.TempDir()
instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
77: {ID: 77, UserID: 20, WorkspacePath: &workspacePath},
}}
fileService := &fakeWorkspaceFileService{}
handler := NewWorkspaceFileHandler(instanceService, fileService)
router := workspaceFileTestRouter(10, "user")
router.GET("/api/v1/instances/:id/workspace/files", handler.List)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files", nil)
router.ServeHTTP(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, body = %s, want 403", rec.Code, rec.Body.String())
}
if fileService.listCalls != 0 {
t.Fatalf("workspace file service called %d times, want 0", fileService.listCalls)
}
}
func TestWorkspaceFileHandlerAdminUsesInstanceOwnerScope(t *testing.T) {
gin.SetMode(gin.TestMode)
workspacePath := t.TempDir()
instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
77: {ID: 77, UserID: 20, WorkspacePath: &workspacePath},
}}
fileService := &fakeWorkspaceFileService{}
handler := NewWorkspaceFileHandler(instanceService, fileService)
router := workspaceFileTestRouter(10, "admin")
router.GET("/api/v1/instances/:id/workspace/files", handler.List)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files?path=", nil)
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
if fileService.lastScope.InstanceID != 77 || fileService.lastScope.UserID != 20 || fileService.lastScope.WorkspacePath != workspacePath {
t.Fatalf("scope = %#v, want instance owner scope", fileService.lastScope)
}
}
func TestWorkspaceFileHandlerRequiresWorkspacePath(t *testing.T) {
gin.SetMode(gin.TestMode)
instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
77: {ID: 77, UserID: 10},
}}
fileService := &fakeWorkspaceFileService{}
handler := NewWorkspaceFileHandler(instanceService, fileService)
router := workspaceFileTestRouter(10, "user")
router.GET("/api/v1/instances/:id/workspace/files", handler.List)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files", nil)
router.ServeHTTP(rec, req)
if rec.Code != http.StatusNotFound {
t.Fatalf("status = %d, body = %s, want 404", rec.Code, rec.Body.String())
}
if fileService.listCalls != 0 {
t.Fatalf("workspace file service called %d times, want 0", fileService.listCalls)
}
}
func TestWorkspaceFileHandlerProDesktopUsesRuntimeConfigWorkspace(t *testing.T) {
gin.SetMode(gin.TestMode)
instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
77: {
ID: 77,
UserID: 10,
RuntimeType: services.RuntimeBackendDesktop,
InstanceMode: services.InstanceModePro,
},
}}
localFileService := &fakeWorkspaceFileService{}
runtimeFileService := &fakeWorkspaceFileService{}
handler := NewWorkspaceFileHandler(instanceService, localFileService, runtimeFileService)
router := workspaceFileTestRouter(10, "user")
router.GET("/api/v1/instances/:id/workspace/files", handler.List)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files?path=", nil)
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
if localFileService.listCalls != 0 {
t.Fatalf("local workspace service called %d times, want 0", localFileService.listCalls)
}
if runtimeFileService.listCalls != 1 {
t.Fatalf("runtime workspace service called %d times, want 1", runtimeFileService.listCalls)
}
if runtimeFileService.lastScope.WorkspacePath != "/config" {
t.Fatalf("runtime workspace path = %q, want /config", runtimeFileService.lastScope.WorkspacePath)
}
}
func TestWorkspaceFileHandlerDownloadUsesSafeContentDisposition(t *testing.T) {
gin.SetMode(gin.TestMode)
workspacePath := t.TempDir()
downloadFile, err := os.CreateTemp(t.TempDir(), "download-*")
if err != nil {
t.Fatal(err)
}
if _, err := downloadFile.WriteString("body"); err != nil {
t.Fatal(err)
}
if _, err := downloadFile.Seek(0, io.SeekStart); err != nil {
t.Fatal(err)
}
instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
77: {ID: 77, UserID: 10, WorkspacePath: &workspacePath},
}}
fileService := &fakeWorkspaceFileService{file: downloadFile, filename: `bad"name/ignored.txt`, size: int64(len("body"))}
handler := NewWorkspaceFileHandler(instanceService, fileService)
router := workspaceFileTestRouter(10, "user")
router.GET("/api/v1/instances/:id/workspace/download", handler.Download)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/download?path=report.txt", nil)
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
disposition := rec.Header().Get("Content-Disposition")
if !strings.Contains(disposition, "attachment;") || strings.Contains(disposition, `"name/`) || strings.Contains(disposition, "\r") || strings.Contains(disposition, "\n") {
t.Fatalf("Content-Disposition = %q, want safe attachment filename", disposition)
}
}
func workspaceFileTestRouter(userID int, role string) *gin.Engine {
router := gin.New()
router.Use(func(c *gin.Context) {
c.Set("userID", userID)
c.Set("userRole", role)
c.Next()
})
return router
}
backend/internal/middleware/auth_middleware.go
+28 -1
View File
@@ -51,7 +51,11 @@ func Auth() gin.HandlerFunc {
}
// GatewayAuth accepts either a normal user access JWT or an instance lifecycle gateway token.
func GatewayAuth(instanceRepo repository.InstanceRepository) gin.HandlerFunc {
func GatewayAuth(instanceRepo repository.InstanceRepository, bindingRepos ...repository.InstanceRuntimeBindingRepository) gin.HandlerFunc {
var bindingRepo repository.InstanceRuntimeBindingRepository
if len(bindingRepos) > 0 {
bindingRepo = bindingRepos[0]
}
return func(c *gin.Context) {
tokenString, ok := extractToken(c)
if !ok {
@@ -90,11 +94,34 @@ func GatewayAuth(instanceRepo repository.InstanceRepository) gin.HandlerFunc {
c.Set("userID", instance.UserID)
c.Set("instanceID", instance.ID)
c.Set("instanceMode", gatewayInstanceMode(instance.InstanceMode, instance.RuntimeType))
c.Set("runtimeType", strings.TrimSpace(instance.RuntimeType))
if bindingRepo != nil {
if binding, err := bindingRepo.GetRunningByInstanceID(c.Request.Context(), instance.ID); err == nil && binding != nil {
c.Set("gatewayID", strings.TrimSpace(binding.GatewayID))
c.Set("runtimePodID", binding.RuntimePodID)
}
}
c.Set("gatewayAuthType", "instance")
c.Next()
}
}
func gatewayInstanceMode(instanceMode, runtimeType string) string {
mode := strings.ToLower(strings.TrimSpace(instanceMode))
if mode == "lite" || mode == "pro" {
return mode
}
switch strings.ToLower(strings.TrimSpace(runtimeType)) {
case "gateway":
return "lite"
case "desktop", "shell":
return "pro"
default:
return mode
}
}
func extractToken(c *gin.Context) (string, bool) {
authHeader := c.GetHeader("Authorization")
if authHeader != "" {
backend/internal/models/audit_event.go
+4
View File
@@ -10,6 +10,10 @@ type AuditEvent struct {
RequestID *string `db:"request_id" json:"request_id,omitempty"`
UserID *int `db:"user_id" json:"user_id,omitempty"`
InstanceID *int `db:"instance_id" json:"instance_id,omitempty"`
InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"`
RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"`
GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"`
RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"`
InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"`
EventType string `db:"event_type" json:"event_type"`
TrafficClass string `db:"traffic_class" json:"traffic_class"`
backend/internal/models/cost_record.go
+4
View File
@@ -10,6 +10,10 @@ type CostRecord struct {
RequestID *string `db:"request_id" json:"request_id,omitempty"`
UserID *int `db:"user_id" json:"user_id,omitempty"`
InstanceID *int `db:"instance_id" json:"instance_id,omitempty"`
InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"`
RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"`
GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"`
RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"`
InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"`
ModelID *int `db:"model_id" json:"model_id,omitempty"`
ProviderType string `db:"provider_type" json:"provider_type"`
backend/internal/models/instance.go
+5
View File
@@ -12,6 +12,7 @@ type Instance struct {
Description *string `db:"description" json:"description,omitempty"`
Type string `db:"type" json:"type"`
RuntimeType string `db:"runtime_type" json:"runtime_type"`
InstanceMode string `db:"instance_mode" json:"instance_mode"`
Status string `db:"status" json:"status"`
CPUCores float64 `db:"cpu_cores" json:"cpu_cores"`
MemoryGB int `db:"memory_gb" json:"memory_gb"`
@@ -26,6 +27,10 @@ type Instance struct {
EnvironmentOverridesJSON *string `db:"environment_overrides_json" json:"-"`
StorageClass string `db:"storage_class" json:"storage_class"`
MountPath string `db:"mount_path" json:"mount_path"`
WorkspacePath *string `db:"workspace_path" json:"workspace_path,omitempty"`
WorkspaceUsageBytes int64 `db:"workspace_usage_bytes" json:"workspace_usage_bytes"`
RuntimeGeneration int `db:"runtime_generation" json:"runtime_generation"`
RuntimeErrorMessage *string `db:"runtime_error_message" json:"runtime_error_message,omitempty"`
PodName *string `db:"pod_name" json:"pod_name,omitempty"`
PodNamespace *string `db:"pod_namespace" json:"pod_namespace,omitempty"`
PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"`
backend/internal/models/instance_external_access.go
+25
View File
@@ -0,0 +1,25 @@
package models
import "time"
type InstanceExternalAccess struct {
ID int64 `db:"id,primarykey,autoincrement" json:"id"`
InstanceID int `db:"instance_id" json:"instance_id"`
Enabled bool `db:"enabled" json:"enabled"`
AuthMode string `db:"auth_mode" json:"auth_mode"`
PublicSlug *string `db:"public_slug" json:"-"`
PublicTokenHash *string `db:"public_token_hash" json:"-"`
ShortCodeHash *string `db:"short_code_hash" json:"-"`
PasswordHash *string `db:"api_key_hash" json:"-"`
PasswordValue *string `db:"password_value" json:"-"`
PasswordHint *string `db:"api_key_prefix" json:"password_hint,omitempty"`
ExpiresAt *time.Time `db:"expires_at" json:"expires_at,omitempty"`
CreatedBy *int `db:"created_by" json:"created_by,omitempty"`
LastUsedAt *time.Time `db:"last_used_at" json:"last_used_at,omitempty"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
}
func (InstanceExternalAccess) TableName() string {
return "instance_external_access"
}
backend/internal/models/instance_runtime_binding.go
+24
View File
@@ -0,0 +1,24 @@
package models
import "time"
type InstanceRuntimeBinding struct {
ID int64 `db:"id,primarykey,autoincrement" json:"id"`
InstanceID int `db:"instance_id" json:"instance_id"`
RuntimePodID int64 `db:"runtime_pod_id" json:"runtime_pod_id"`
RuntimeType string `db:"runtime_type" json:"runtime_type"`
GatewayID string `db:"gateway_id" json:"gateway_id"`
GatewayPort int `db:"gateway_port" json:"gateway_port"`
GatewayPID *int `db:"gateway_pid" json:"gateway_pid,omitempty"`
WorkspacePath string `db:"workspace_path" json:"workspace_path"`
State string `db:"state" json:"state"`
Generation int `db:"generation" json:"generation"`
LastHealthAt *time.Time `db:"last_health_at" json:"last_health_at,omitempty"`
ErrorMessage *string `db:"error_message" json:"error_message,omitempty"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
}
func (InstanceRuntimeBinding) TableName() string {
return "instance_runtime_bindings"
}
backend/internal/models/model_invocation.go
+4
View File
@@ -10,6 +10,10 @@ type ModelInvocation struct {
RequestID string `db:"request_id" json:"request_id"`
UserID *int `db:"user_id" json:"user_id,omitempty"`
InstanceID *int `db:"instance_id" json:"instance_id,omitempty"`
InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"`
RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"`
GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"`
RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"`
ModelID *int `db:"model_id" json:"model_id,omitempty"`
ProviderType string `db:"provider_type" json:"provider_type"`
RequestedModel string `db:"requested_model" json:"requested_model"`
backend/internal/models/risk_hit.go
+4
View File
@@ -10,6 +10,10 @@ type RiskHit struct {
RequestID *string `db:"request_id" json:"request_id,omitempty"`
UserID *int `db:"user_id" json:"user_id,omitempty"`
InstanceID *int `db:"instance_id" json:"instance_id,omitempty"`
InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"`
RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"`
GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"`
RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"`
InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"`
RuleID string `db:"rule_id" json:"rule_id"`
RuleName string `db:"rule_name" json:"rule_name"`
backend/internal/models/runtime_pod.go
+33
View File
@@ -0,0 +1,33 @@
package models
import "time"
type RuntimePod struct {
ID int64 `db:"id,primarykey,autoincrement" json:"id"`
RuntimeType string `db:"runtime_type" json:"runtime_type"`
Namespace string `db:"namespace" json:"namespace"`
PodName string `db:"pod_name" json:"pod_name"`
PodUID *string `db:"pod_uid" json:"pod_uid,omitempty"`
PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"`
NodeName *string `db:"node_name" json:"node_name,omitempty"`
DeploymentName string `db:"deployment_name" json:"deployment_name"`
ImageRef string `db:"image_ref" json:"image_ref"`
AgentEndpoint *string `db:"agent_endpoint" json:"agent_endpoint,omitempty"`
State string `db:"state" json:"state"`
Capacity int `db:"capacity" json:"capacity"`
UsedSlots int `db:"used_slots" json:"used_slots"`
Draining bool `db:"draining" json:"draining"`
CPUMillisUsed int64 `db:"cpu_millis_used" json:"cpu_millis_used"`
MemoryBytesUsed int64 `db:"memory_bytes_used" json:"memory_bytes_used"`
DiskBytesUsed int64 `db:"disk_bytes_used" json:"disk_bytes_used"`
NetworkRXBytes int64 `db:"network_rx_bytes" json:"network_rx_bytes"`
NetworkTXBytes int64 `db:"network_tx_bytes" json:"network_tx_bytes"`
MetricsJSON *string `db:"metrics_json" json:"-"`
LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
}
func (RuntimePod) TableName() string {
return "runtime_pods"
}
backend/internal/models/runtime_rollout.go
+22
View File
@@ -0,0 +1,22 @@
package models
import "time"
type RuntimeRollout struct {
ID int64 `db:"id,primarykey,autoincrement" json:"id"`
RuntimeType string `db:"runtime_type" json:"runtime_type"`
TargetImageRef string `db:"target_image_ref" json:"target_image_ref"`
Status string `db:"status" json:"status"`
BatchSize int `db:"batch_size" json:"batch_size"`
MaxUnavailable int `db:"max_unavailable" json:"max_unavailable"`
StartedBy *int `db:"started_by" json:"started_by,omitempty"`
StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"`
FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"`
ErrorMessage *string `db:"error_message" json:"error_message,omitempty"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
}
func (RuntimeRollout) TableName() string {
return "runtime_rollouts"
}
backend/internal/models/team.go
+1
View File
@@ -65,6 +65,7 @@ type TeamMember struct {
DisplayName string `db:"display_name" json:"display_name"`
Role string `db:"role" json:"role"`
RuntimeType string `db:"runtime_type" json:"runtime_type"`
InstanceMode string `db:"instance_mode" json:"instance_mode"`
Description *string `db:"description" json:"description,omitempty"`
Status string `db:"status" json:"status"`
CurrentTaskID *int `db:"current_task_id" json:"current_task_id,omitempty"`
backend/internal/models/workspace_file_audit.go
+17
View File
@@ -0,0 +1,17 @@
package models
import "time"
type WorkspaceFileAudit struct {
ID int64 `db:"id,primarykey,autoincrement" json:"id"`
InstanceID int `db:"instance_id" json:"instance_id"`
UserID int `db:"user_id" json:"user_id"`
Action string `db:"action" json:"action"`
RelativePath string `db:"relative_path" json:"relative_path"`
Bytes int64 `db:"bytes" json:"bytes"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
}
func (WorkspaceFileAudit) TableName() string {
return "workspace_file_audits"
}
backend/internal/repository/audit_event_repository.go
+5
View File
@@ -36,6 +36,10 @@ CREATE TABLE IF NOT EXISTS audit_events (
request_id VARCHAR(100) NULL,
user_id INT NULL,
instance_id INT NULL,
instance_mode VARCHAR(16) NULL,
runtime_type VARCHAR(32) NULL,
gateway_id VARCHAR(128) NULL,
runtime_pod_id BIGINT NULL,
invocation_id INT NULL,
event_type VARCHAR(100) NOT NULL,
traffic_class VARCHAR(50) NOT NULL,
@@ -46,6 +50,7 @@ CREATE TABLE IF NOT EXISTS audit_events (
INDEX idx_audit_events_trace_id (trace_id),
INDEX idx_audit_events_request_id (request_id),
INDEX idx_audit_events_user_id (user_id),
INDEX idx_audit_events_gateway_id (gateway_id),
INDEX idx_audit_events_invocation_id (invocation_id),
INDEX idx_audit_events_event_type (event_type),
INDEX idx_audit_events_created_at (created_at)
backend/internal/repository/cost_record_repository.go
+5
View File
@@ -37,6 +37,10 @@ CREATE TABLE IF NOT EXISTS cost_records (
request_id VARCHAR(100) NULL,
user_id INT NULL,
instance_id INT NULL,
instance_mode VARCHAR(16) NULL,
runtime_type VARCHAR(32) NULL,
gateway_id VARCHAR(128) NULL,
runtime_pod_id BIGINT NULL,
invocation_id INT NULL,
model_id INT NULL,
provider_type VARCHAR(100) NOT NULL,
@@ -53,6 +57,7 @@ CREATE TABLE IF NOT EXISTS cost_records (
recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
INDEX idx_cost_records_trace_id (trace_id),
INDEX idx_cost_records_user_id (user_id),
INDEX idx_cost_records_gateway_id (gateway_id),
INDEX idx_cost_records_model_id (model_id),
INDEX idx_cost_records_provider_type (provider_type),
INDEX idx_cost_records_recorded_at (recorded_at)
backend/internal/repository/instance_external_access_repository.go
+107
View File
@@ -0,0 +1,107 @@
package repository
import (
"context"
"fmt"
"time"
"clawreef/internal/models"
"github.com/upper/db/v4"
)
type InstanceExternalAccessRepository interface {
GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error)
GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error)
Upsert(ctx context.Context, access *models.InstanceExternalAccess) error
Disable(ctx context.Context, instanceID int) error
MarkUsed(ctx context.Context, id int64) error
}
type instanceExternalAccessRepository struct {
sess db.Session
}
func NewInstanceExternalAccessRepository(sess db.Session) InstanceExternalAccessRepository {
return &instanceExternalAccessRepository{sess: sess}
}
func (r *instanceExternalAccessRepository) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) {
_ = ctx
var access models.InstanceExternalAccess
if err := r.sess.Collection(access.TableName()).Find(db.Cond{"instance_id": instanceID}).One(&access); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get instance external access: %w", err)
}
return &access, nil
}
func (r *instanceExternalAccessRepository) GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) {
_ = ctx
var access models.InstanceExternalAccess
if err := r.sess.Collection(access.TableName()).Find(db.Cond{"short_code_hash": codeHash}).One(&access); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get instance external access by short code hash: %w", err)
}
return &access, nil
}
func (r *instanceExternalAccessRepository) Upsert(ctx context.Context, access *models.InstanceExternalAccess) error {
now := time.Now().UTC()
if access.CreatedAt.IsZero() {
access.CreatedAt = now
}
access.UpdatedAt = now
_, err := r.sess.SQL().ExecContext(ctx, `
INSERT INTO instance_external_access (
instance_id, enabled, auth_mode, public_slug, short_code_hash, public_token_hash,
api_key_hash, password_value, api_key_prefix, expires_at, created_by,
created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
enabled = VALUES(enabled),
auth_mode = VALUES(auth_mode),
public_slug = VALUES(public_slug),
short_code_hash = VALUES(short_code_hash),
public_token_hash = VALUES(public_token_hash),
api_key_hash = VALUES(api_key_hash),
password_value = VALUES(password_value),
api_key_prefix = VALUES(api_key_prefix),
expires_at = VALUES(expires_at),
created_by = VALUES(created_by),
updated_at = VALUES(updated_at)
`, access.InstanceID, access.Enabled, access.AuthMode, access.PublicSlug, access.ShortCodeHash, access.PublicTokenHash,
access.PasswordHash, access.PasswordValue, access.PasswordHint, access.ExpiresAt, access.CreatedBy,
access.CreatedAt, access.UpdatedAt)
if err != nil {
return fmt.Errorf("failed to upsert instance external access: %w", err)
}
return nil
}
func (r *instanceExternalAccessRepository) Disable(ctx context.Context, instanceID int) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE instance_external_access
SET enabled = 0, updated_at = ?
WHERE instance_id = ?
`, time.Now().UTC(), instanceID)
if err != nil {
return fmt.Errorf("failed to disable instance external access: %w", err)
}
return nil
}
func (r *instanceExternalAccessRepository) MarkUsed(ctx context.Context, id int64) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE instance_external_access
SET last_used_at = ?, updated_at = ?
WHERE id = ?
`, time.Now().UTC(), time.Now().UTC(), id)
if err != nil {
return fmt.Errorf("failed to mark instance external access used: %w", err)
}
return nil
}
backend/internal/repository/instance_repository.go
+166
View File
@@ -1,13 +1,19 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
"clawreef/internal/models"
"github.com/upper/db/v4"
)
var ErrStaleRuntimeGeneration = errors.New("stale runtime generation")
// InstanceRepository defines the interface for instance data operations
type InstanceRepository interface {
Create(instance *models.Instance) error
@@ -18,8 +24,14 @@ type InstanceRepository interface {
CountAll() (int, error)
GetByUserID(userID int, offset, limit int) ([]models.Instance, error)
CountByUserID(userID int) (int, error)
CountActiveByMode(ctx context.Context, mode string) (int, error)
ExistsByUserIDAndName(userID int, name string) (bool, error)
GetAllRunning() ([]models.Instance, error)
GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error)
GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error)
UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error
SetWorkspacePath(ctx context.Context, id int, workspacePath string) error
UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error
Update(instance *models.Instance) error
Delete(id int) error
}
@@ -121,6 +133,27 @@ func (r *instanceRepository) CountByUserID(userID int) (int, error) {
return int(count), nil
}
func (r *instanceRepository) CountActiveByMode(ctx context.Context, mode string) (int, error) {
normalized := strings.TrimSpace(strings.ToLower(mode))
if normalized == "" {
return 0, nil
}
row, err := r.sess.SQL().QueryRowContext(ctx, `
SELECT COUNT(*)
FROM instances
WHERE instance_mode = ?
AND status IN ('creating', 'running')
`, normalized)
if err != nil {
return 0, fmt.Errorf("failed to count active instances by mode: %w", err)
}
var count int
if err := row.Scan(&count); err != nil {
return 0, fmt.Errorf("failed to scan active instances by mode count: %w", err)
}
return count, nil
}
// ExistsByUserIDAndName checks whether the user already has an instance with the same display name.
func (r *instanceRepository) ExistsByUserIDAndName(userID int, name string) (bool, error) {
instances, err := r.GetByUserID(userID, 0, 1000)
@@ -155,6 +188,139 @@ func (r *instanceRepository) GetAllRunning() ([]models.Instance, error) {
return instances, nil
}
func (r *instanceRepository) GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if limit <= 0 {
limit = 100
}
var instances []models.Instance
query, args := buildV2SchedulerInstanceQuery(v2DesiredRunningStatuses(), limit)
iter := r.sess.SQL().IteratorContext(ctx, query, args...)
defer iter.Close()
if err := iter.All(&instances); err != nil {
return nil, fmt.Errorf("failed to get v2 desired running instances: %w", err)
}
return instances, nil
}
func v2DesiredRunningStatuses() []string {
return []string{"creating", "running", "error"}
}
func (r *instanceRepository) GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if limit <= 0 {
limit = 100
}
var instances []models.Instance
query, args := buildV2SchedulerInstanceQuery([]string{"creating"}, limit)
iter := r.sess.SQL().IteratorContext(ctx, query, args...)
defer iter.Close()
if err := iter.All(&instances); err != nil {
return nil, fmt.Errorf("failed to get v2 creating instances: %w", err)
}
return instances, nil
}
func buildV2SchedulerInstanceQuery(statuses []string, limit int) (string, []any) {
if limit <= 0 {
limit = 100
}
if len(statuses) == 0 {
statuses = []string{"creating", "running"}
}
statusPlaceholders := strings.TrimRight(strings.Repeat("?, ", len(statuses)), ", ")
args := make([]any, 0, len(statuses)+5)
for _, status := range statuses {
args = append(args, status)
}
args = append(args, "gateway", "lite", "openclaw", "hermes", limit)
return fmt.Sprintf(`
SELECT *
FROM instances
WHERE status IN (%s)
AND runtime_type = ?
AND instance_mode = ?
AND workspace_path IS NOT NULL
AND TRIM(workspace_path) <> ''
AND type IN (?, ?)
ORDER BY id
LIMIT ?
`, statusPlaceholders), args
}
func (r *instanceRepository) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error {
res, err := r.sess.SQL().ExecContext(ctx, `
UPDATE instances
SET status = ?, runtime_generation = ?, runtime_error_message = ?, updated_at = ?
WHERE id = ? AND runtime_generation <= ?
`, status, generation, message, time.Now().UTC(), id, generation)
if err != nil {
return fmt.Errorf("failed to update instance runtime state: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("failed to inspect instance runtime state update: %w", err)
}
if affected == 0 {
currentGeneration, err := r.getRuntimeGeneration(ctx, id)
if err != nil {
return err
}
if currentGeneration > generation {
return ErrStaleRuntimeGeneration
}
}
return nil
}
func (r *instanceRepository) getRuntimeGeneration(ctx context.Context, id int) (int, error) {
var currentGeneration int
row, err := r.sess.SQL().QueryRowContext(ctx, `
SELECT runtime_generation
FROM instances
WHERE id = ?
`, id)
if err != nil {
return 0, fmt.Errorf("failed to query instance runtime generation: %w", err)
}
if err := row.Scan(&currentGeneration); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return 0, ErrStaleRuntimeGeneration
}
return 0, fmt.Errorf("failed to scan instance runtime generation: %w", err)
}
return currentGeneration, nil
}
func (r *instanceRepository) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE instances
SET workspace_path = ?, updated_at = ?
WHERE id = ?
`, workspacePath, time.Now().UTC(), id)
if err != nil {
return fmt.Errorf("failed to set instance workspace path: %w", err)
}
return nil
}
func (r *instanceRepository) UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE instances
SET workspace_usage_bytes = ?, updated_at = ?
WHERE id = ?
`, usageBytes, time.Now().UTC(), id)
if err != nil {
return fmt.Errorf("failed to update instance workspace usage: %w", err)
}
return nil
}
// Update updates an instance
func (r *instanceRepository) Update(instance *models.Instance) error {
err := r.sess.Collection("instances").Find(db.Cond{"id": instance.ID}).Update(instance)
backend/internal/repository/instance_repository_test.go
+58
View File
@@ -0,0 +1,58 @@
package repository
import (
"reflect"
"strings"
"testing"
)
func TestBuildV2SchedulerInstanceQueryRequiresWorkspaceTypeAndStatuses(t *testing.T) {
query, args := buildV2SchedulerInstanceQuery([]string{"creating", "running"}, 25)
normalized := normalizeSQLForTest(query)
requiredFragments := []string{
"FROM instances",
"status IN (?, ?)",
"runtime_type = ?",
"instance_mode = ?",
"workspace_path IS NOT NULL",
"TRIM(workspace_path) <> ''",
"type IN (?, ?)",
"ORDER BY id",
"LIMIT ?",
}
for _, fragment := range requiredFragments {
if !strings.Contains(normalized, fragment) {
t.Fatalf("query %q does not contain %q", normalized, fragment)
}
}
wantArgs := []any{"creating", "running", "gateway", "lite", "openclaw", "hermes", 25}
if !reflect.DeepEqual(args, wantArgs) {
t.Fatalf("args = %#v, want %#v", args, wantArgs)
}
}
func TestV2DesiredRunningStatusesIncludeRecoverableErrorCandidates(t *testing.T) {
want := []string{"creating", "running", "error"}
if got := v2DesiredRunningStatuses(); !reflect.DeepEqual(got, want) {
t.Fatalf("desired running statuses = %#v, want %#v", got, want)
}
}
func TestBuildV2SchedulerInstanceQueryDefaultsLimit(t *testing.T) {
query, args := buildV2SchedulerInstanceQuery([]string{"creating"}, 0)
normalized := normalizeSQLForTest(query)
if !strings.Contains(normalized, "status IN (?)") {
t.Fatalf("query %q does not contain single status predicate", normalized)
}
wantArgs := []any{"creating", "gateway", "lite", "openclaw", "hermes", 100}
if !reflect.DeepEqual(args, wantArgs) {
t.Fatalf("args = %#v, want %#v", args, wantArgs)
}
}
func normalizeSQLForTest(query string) string {
return strings.Join(strings.Fields(query), " ")
}
backend/internal/repository/instance_runtime_binding_repository.go
+213
View File
@@ -0,0 +1,213 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"clawreef/internal/models"
"github.com/upper/db/v4"
)
type InstanceRuntimeBindingRepository interface {
Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error
GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error)
GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error)
ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error)
ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error)
UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error
UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error
DeleteByInstanceID(ctx context.Context, instanceID int) error
DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error
}
type instanceRuntimeBindingRepository struct {
sess db.Session
}
func NewInstanceRuntimeBindingRepository(sess db.Session) InstanceRuntimeBindingRepository {
return &instanceRuntimeBindingRepository{sess: sess}
}
func (r *instanceRuntimeBindingRepository) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
if err := ctx.Err(); err != nil {
return err
}
ensureTimestamps(&binding.CreatedAt, &binding.UpdatedAt)
res, err := r.sess.Collection("instance_runtime_bindings").Insert(binding)
if err != nil {
return fmt.Errorf("failed to create instance runtime binding: %w", err)
}
if id, ok := res.ID().(int64); ok {
binding.ID = id
}
return nil
}
func (r *instanceRuntimeBindingRepository) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var binding models.InstanceRuntimeBinding
if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"instance_id": instanceID}).One(&binding); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get instance runtime binding: %w", err)
}
return &binding, nil
}
func (r *instanceRuntimeBindingRepository) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var binding models.InstanceRuntimeBinding
if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"instance_id": instanceID, "state": "running"}).One(&binding); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get running instance runtime binding: %w", err)
}
return &binding, nil
}
func (r *instanceRuntimeBindingRepository) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var bindings []models.InstanceRuntimeBinding
if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"runtime_pod_id": runtimePodID}).OrderBy("id").All(&bindings); err != nil {
return nil, fmt.Errorf("failed to list instance runtime bindings: %w", err)
}
return bindings, nil
}
func (r *instanceRuntimeBindingRepository) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if len(runtimePodIDs) == 0 {
return []models.InstanceRuntimeBinding{}, nil
}
var bindings []models.InstanceRuntimeBinding
if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"runtime_pod_id IN": runtimePodIDs}).OrderBy("runtime_pod_id", "id").All(&bindings); err != nil {
return nil, fmt.Errorf("failed to list instance runtime bindings by pods: %w", err)
}
return bindings, nil
}
func (r *instanceRuntimeBindingRepository) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error {
now := time.Now().UTC()
res, err := r.sess.SQL().ExecContext(ctx, `
UPDATE instance_runtime_bindings
SET state = 'running', generation = ?, gateway_id = ?, gateway_port = ?, gateway_pid = ?,
last_health_at = ?, error_message = NULL, updated_at = ?
WHERE instance_id = ? AND generation <= ?
`, generation, gatewayID, port, pid, now, now, instanceID, generation)
if err != nil {
return fmt.Errorf("failed to update running instance runtime binding: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("failed to inspect running instance runtime binding update: %w", err)
}
if affected == 0 {
currentGeneration, err := r.getGeneration(ctx, instanceID)
if err != nil {
return err
}
if currentGeneration > generation {
return ErrStaleRuntimeGeneration
}
}
return nil
}
func (r *instanceRuntimeBindingRepository) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
res, err := r.sess.SQL().ExecContext(ctx, `
UPDATE instance_runtime_bindings
SET state = ?, generation = ?, error_message = ?, updated_at = ?
WHERE instance_id = ? AND generation <= ?
`, state, generation, message, time.Now().UTC(), instanceID, generation)
if err != nil {
return fmt.Errorf("failed to update instance runtime binding state: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("failed to inspect instance runtime binding state update: %w", err)
}
if affected == 0 {
currentGeneration, err := r.getGeneration(ctx, instanceID)
if err != nil {
return err
}
if currentGeneration > generation {
return ErrStaleRuntimeGeneration
}
}
return nil
}
func (r *instanceRuntimeBindingRepository) getGeneration(ctx context.Context, instanceID int) (int, error) {
var currentGeneration int
row, err := r.sess.SQL().QueryRowContext(ctx, `
SELECT generation
FROM instance_runtime_bindings
WHERE instance_id = ?
`, instanceID)
if err != nil {
return 0, fmt.Errorf("failed to query instance runtime binding generation: %w", err)
}
if err := row.Scan(&currentGeneration); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return 0, ErrStaleRuntimeGeneration
}
return 0, fmt.Errorf("failed to scan instance runtime binding generation: %w", err)
}
return currentGeneration, nil
}
func (r *instanceRuntimeBindingRepository) DeleteByInstanceID(ctx context.Context, instanceID int) error {
_, err := r.sess.SQL().ExecContext(ctx, `
DELETE FROM instance_runtime_bindings
WHERE instance_id = ?
`, instanceID)
if err != nil {
return fmt.Errorf("failed to delete instance runtime binding: %w", err)
}
return nil
}
func (r *instanceRuntimeBindingRepository) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
if err := ctx.Err(); err != nil {
return err
}
return r.sess.TxContext(ctx, func(tx db.Session) error {
res, err := tx.SQL().ExecContext(ctx, `
DELETE FROM instance_runtime_bindings
WHERE instance_id = ? AND runtime_pod_id = ?
`, instanceID, runtimePodID)
if err != nil {
return fmt.Errorf("failed to delete instance runtime binding: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("failed to inspect instance runtime binding delete: %w", err)
}
if affected == 0 {
return nil
}
if _, err := tx.SQL().ExecContext(ctx, `
UPDATE runtime_pods
SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ?
WHERE id = ?
`, time.Now().UTC(), runtimePodID); err != nil {
return fmt.Errorf("failed to release runtime pod slot: %w", err)
}
return nil
}, nil)
}
backend/internal/repository/model_invocation_repository.go
+5
View File
@@ -40,6 +40,10 @@ CREATE TABLE IF NOT EXISTS model_invocations (
request_id VARCHAR(100) NOT NULL,
user_id INT NULL,
instance_id INT NULL,
instance_mode VARCHAR(16) NULL,
runtime_type VARCHAR(32) NULL,
gateway_id VARCHAR(128) NULL,
runtime_pod_id BIGINT NULL,
model_id INT NULL,
provider_type VARCHAR(100) NOT NULL,
requested_model VARCHAR(255) NOT NULL,
@@ -62,6 +66,7 @@ CREATE TABLE IF NOT EXISTS model_invocations (
INDEX idx_model_invocations_request_id (request_id),
INDEX idx_model_invocations_user_id (user_id),
INDEX idx_model_invocations_instance_id (instance_id),
INDEX idx_model_invocations_gateway_id (gateway_id),
INDEX idx_model_invocations_model_id (model_id),
INDEX idx_model_invocations_status (status),
INDEX idx_model_invocations_created_at (created_at)
backend/internal/repository/risk_hit_repository.go
+5
View File
@@ -35,6 +35,10 @@ CREATE TABLE IF NOT EXISTS risk_hits (
request_id VARCHAR(100) NULL,
user_id INT NULL,
instance_id INT NULL,
instance_mode VARCHAR(16) NULL,
runtime_type VARCHAR(32) NULL,
gateway_id VARCHAR(128) NULL,
runtime_pod_id BIGINT NULL,
invocation_id INT NULL,
rule_id VARCHAR(100) NOT NULL,
rule_name VARCHAR(255) NOT NULL,
@@ -45,6 +49,7 @@ CREATE TABLE IF NOT EXISTS risk_hits (
INDEX idx_risk_hits_trace_id (trace_id),
INDEX idx_risk_hits_request_id (request_id),
INDEX idx_risk_hits_user_id (user_id),
INDEX idx_risk_hits_gateway_id (gateway_id),
INDEX idx_risk_hits_invocation_id (invocation_id),
INDEX idx_risk_hits_severity (severity),
INDEX idx_risk_hits_created_at (created_at)
backend/internal/repository/runtime_pod_repository.go
+224
View File
@@ -0,0 +1,224 @@
package repository
import (
"context"
"fmt"
"time"
"clawreef/internal/models"
"github.com/upper/db/v4"
)
type RuntimePodMetricsUpdate struct {
CPUMillisUsed int64
MemoryBytesUsed int64
DiskBytesUsed int64
NetworkRXBytes int64
NetworkTXBytes int64
MetricsJSON *string
LastSeenAt *time.Time
}
type RuntimePodRepository interface {
UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error
GetByID(ctx context.Context, id int64) (*models.RuntimePod, error)
GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error)
List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error)
ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error)
TryClaimSlot(ctx context.Context, podID int64) (bool, error)
ReleaseSlot(ctx context.Context, podID int64) error
MarkState(ctx context.Context, podID int64, state string, draining bool) error
UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error
MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error
UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error
}
type runtimePodRepository struct {
sess db.Session
}
func NewRuntimePodRepository(sess db.Session) RuntimePodRepository {
return &runtimePodRepository{sess: sess}
}
func (r *runtimePodRepository) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error {
ensureTimestamps(&pod.CreatedAt, &pod.UpdatedAt)
res, err := r.sess.SQL().ExecContext(ctx, `
INSERT INTO runtime_pods (
runtime_type, namespace, pod_name, pod_uid, pod_ip, node_name, deployment_name,
image_ref, agent_endpoint, state, capacity, used_slots, draining, cpu_millis_used,
memory_bytes_used, disk_bytes_used, network_rx_bytes, network_tx_bytes, metrics_json,
last_seen_at, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
id = LAST_INSERT_ID(id),
runtime_type = VALUES(runtime_type),
pod_uid = VALUES(pod_uid),
pod_ip = VALUES(pod_ip),
node_name = VALUES(node_name),
deployment_name = VALUES(deployment_name),
image_ref = VALUES(image_ref),
agent_endpoint = VALUES(agent_endpoint),
capacity = VALUES(capacity),
cpu_millis_used = VALUES(cpu_millis_used),
memory_bytes_used = VALUES(memory_bytes_used),
disk_bytes_used = VALUES(disk_bytes_used),
network_rx_bytes = VALUES(network_rx_bytes),
network_tx_bytes = VALUES(network_tx_bytes),
metrics_json = VALUES(metrics_json),
last_seen_at = VALUES(last_seen_at),
updated_at = VALUES(updated_at)
`, pod.RuntimeType, pod.Namespace, pod.PodName, pod.PodUID, pod.PodIP, pod.NodeName, pod.DeploymentName,
pod.ImageRef, pod.AgentEndpoint, pod.State, pod.Capacity, pod.UsedSlots, pod.Draining, pod.CPUMillisUsed,
pod.MemoryBytesUsed, pod.DiskBytesUsed, pod.NetworkRXBytes, pod.NetworkTXBytes, pod.MetricsJSON,
pod.LastSeenAt, pod.CreatedAt, pod.UpdatedAt)
if err != nil {
return fmt.Errorf("failed to upsert runtime pod: %w", err)
}
if id, err := res.LastInsertId(); err == nil {
pod.ID = id
}
return nil
}
func (r *runtimePodRepository) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var pod models.RuntimePod
if err := r.sess.Collection("runtime_pods").Find(db.Cond{"id": id}).One(&pod); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get runtime pod: %w", err)
}
return &pod, nil
}
func (r *runtimePodRepository) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var pod models.RuntimePod
if err := r.sess.Collection("runtime_pods").Find(db.Cond{"namespace": namespace, "pod_name": podName}).One(&pod); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get runtime pod by namespace/name: %w", err)
}
return &pod, nil
}
func (r *runtimePodRepository) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
cond := db.Cond{}
if runtimeType != "" {
cond["runtime_type"] = runtimeType
}
var pods []models.RuntimePod
if err := r.sess.Collection("runtime_pods").Find(cond).OrderBy("namespace", "pod_name").All(&pods); err != nil {
return nil, fmt.Errorf("failed to list runtime pods: %w", err)
}
return pods, nil
}
func (r *runtimePodRepository) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var pods []models.RuntimePod
iter := r.sess.SQL().IteratorContext(ctx, `
SELECT *
FROM runtime_pods
WHERE runtime_type = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity
ORDER BY used_slots, id
`, runtimeType)
defer iter.Close()
if err := iter.All(&pods); err != nil {
return nil, fmt.Errorf("failed to list schedulable runtime pods: %w", err)
}
return pods, nil
}
func (r *runtimePodRepository) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
res, err := r.sess.SQL().ExecContext(ctx, `
UPDATE runtime_pods
SET used_slots = used_slots + 1, updated_at = ?
WHERE id = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity
`, time.Now().UTC(), podID)
if err != nil {
return false, fmt.Errorf("failed to claim runtime pod slot: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return false, fmt.Errorf("failed to inspect runtime pod slot claim: %w", err)
}
return affected == 1, nil
}
func (r *runtimePodRepository) ReleaseSlot(ctx context.Context, podID int64) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE runtime_pods
SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ?
WHERE id = ?
`, time.Now().UTC(), podID)
if err != nil {
return fmt.Errorf("failed to release runtime pod slot: %w", err)
}
return nil
}
func (r *runtimePodRepository) MarkState(ctx context.Context, podID int64, state string, draining bool) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE runtime_pods
SET state = ?, draining = ?, updated_at = ?
WHERE id = ?
`, state, draining, time.Now().UTC(), podID)
if err != nil {
return fmt.Errorf("failed to mark runtime pod state: %w", err)
}
return nil
}
func (r *runtimePodRepository) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE runtime_pods
SET state = ?, used_slots = ?, capacity = ?, draining = ?, last_seen_at = ?, updated_at = ?
WHERE id = ?
`, state, usedSlots, capacity, draining, lastSeenAt, time.Now().UTC(), podID)
if err != nil {
return fmt.Errorf("failed to update runtime pod heartbeat: %w", err)
}
return nil
}
func (r *runtimePodRepository) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE runtime_pods
SET state = 'unhealthy', updated_at = ?
WHERE (last_seen_at IS NULL OR last_seen_at < ?) AND state <> 'unhealthy'
`, time.Now().UTC(), cutoff)
if err != nil {
return fmt.Errorf("failed to mark unseen runtime pods unhealthy: %w", err)
}
return nil
}
func (r *runtimePodRepository) UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE runtime_pods
SET cpu_millis_used = ?, memory_bytes_used = ?, disk_bytes_used = ?,
network_rx_bytes = ?, network_tx_bytes = ?, metrics_json = ?,
last_seen_at = ?, updated_at = ?
WHERE id = ?
`, metrics.CPUMillisUsed, metrics.MemoryBytesUsed, metrics.DiskBytesUsed,
metrics.NetworkRXBytes, metrics.NetworkTXBytes, metrics.MetricsJSON,
metrics.LastSeenAt, time.Now().UTC(), podID)
if err != nil {
return fmt.Errorf("failed to update runtime pod metrics: %w", err)
}
return nil
}
backend/internal/repository/runtime_rollout_repository.go
+82
View File
@@ -0,0 +1,82 @@
package repository
import (
"context"
"fmt"
"time"
"clawreef/internal/models"
"github.com/upper/db/v4"
)
type RuntimeRolloutRepository interface {
Create(ctx context.Context, rollout *models.RuntimeRollout) error
GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error)
ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error)
UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error
}
type runtimeRolloutRepository struct {
sess db.Session
}
func NewRuntimeRolloutRepository(sess db.Session) RuntimeRolloutRepository {
return &runtimeRolloutRepository{sess: sess}
}
func (r *runtimeRolloutRepository) Create(ctx context.Context, rollout *models.RuntimeRollout) error {
if err := ctx.Err(); err != nil {
return err
}
ensureTimestamps(&rollout.CreatedAt, &rollout.UpdatedAt)
res, err := r.sess.Collection("runtime_rollouts").Insert(rollout)
if err != nil {
return fmt.Errorf("failed to create runtime rollout: %w", err)
}
if id, ok := res.ID().(int64); ok {
rollout.ID = id
}
return nil
}
func (r *runtimeRolloutRepository) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var rollout models.RuntimeRollout
if err := r.sess.Collection("runtime_rollouts").Find(db.Cond{"id": id}).One(&rollout); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get runtime rollout: %w", err)
}
return &rollout, nil
}
func (r *runtimeRolloutRepository) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
cond := db.Cond{"status IN": []string{"pending", "running"}}
if runtimeType != "" {
cond["runtime_type"] = runtimeType
}
var rollouts []models.RuntimeRollout
if err := r.sess.Collection("runtime_rollouts").Find(cond).OrderBy("created_at", "id").All(&rollouts); err != nil {
return nil, fmt.Errorf("failed to list active runtime rollouts: %w", err)
}
return rollouts, nil
}
func (r *runtimeRolloutRepository) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error {
_, err := r.sess.SQL().ExecContext(ctx, `
UPDATE runtime_rollouts
SET status = ?, started_at = ?, finished_at = ?, error_message = ?, updated_at = ?
WHERE id = ?
`, status, startedAt, finishedAt, message, time.Now().UTC(), id)
if err != nil {
return fmt.Errorf("failed to update runtime rollout status: %w", err)
}
return nil
}
backend/internal/repository/system_image_setting_repository.go
+26 -2
View File
@@ -36,7 +36,7 @@ func (r *systemImageSettingRepository) ensureTable() {
CREATE TABLE IF NOT EXISTS system_image_settings (
id INT AUTO_INCREMENT PRIMARY KEY,
instance_type VARCHAR(50) NOT NULL,
runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop',
runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop',
display_name VARCHAR(255) NOT NULL,
image VARCHAR(500) NOT NULL,
is_enabled BOOLEAN NOT NULL DEFAULT TRUE,
@@ -52,6 +52,7 @@ CREATE TABLE IF NOT EXISTS system_image_settings (
r.ensureIsEnabledColumn()
r.ensureRuntimeTypeColumn()
r.ensureRuntimeTypeAllowsGateway()
r.ensureInstanceTypeIsNotUnique()
r.ensureInstanceTypeIndex()
}
@@ -96,12 +97,35 @@ WHERE table_schema = DATABASE()
}
if count == 0 {
if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop' AFTER instance_type"); err != nil {
if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop' AFTER instance_type"); err != nil {
panic(fmt.Errorf("failed to ensure system_image_settings.runtime_type column: %w", err))
}
}
}
func (r *systemImageSettingRepository) ensureRuntimeTypeAllowsGateway() {
var columnType string
row, err := r.sess.SQL().QueryRow(`
SELECT COLUMN_TYPE
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = 'system_image_settings'
AND column_name = 'runtime_type'
`)
if err != nil {
panic(fmt.Errorf("failed to inspect system_image_settings runtime_type enum: %w", err))
}
if err := row.Scan(&columnType); err != nil {
panic(fmt.Errorf("failed to scan system_image_settings runtime_type enum: %w", err))
}
if !strings.Contains(columnType, "'gateway'") {
if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'"); err != nil {
panic(fmt.Errorf("failed to allow system_image_settings.runtime_type gateway: %w", err))
}
}
}
func (r *systemImageSettingRepository) ensureInstanceTypeIsNotUnique() {
rows, err := r.sess.SQL().Query(`
SELECT DISTINCT INDEX_NAME
backend/internal/repository/workspace_file_audit_repository.go
+40
View File
@@ -0,0 +1,40 @@
package repository
import (
"context"
"fmt"
"time"
"clawreef/internal/models"
"github.com/upper/db/v4"
)
type WorkspaceFileAuditRepository interface {
Create(ctx context.Context, audit *models.WorkspaceFileAudit) error
}
type workspaceFileAuditRepository struct {
sess db.Session
}
func NewWorkspaceFileAuditRepository(sess db.Session) WorkspaceFileAuditRepository {
return &workspaceFileAuditRepository{sess: sess}
}
func (r *workspaceFileAuditRepository) Create(ctx context.Context, audit *models.WorkspaceFileAudit) error {
if err := ctx.Err(); err != nil {
return err
}
if audit.CreatedAt.IsZero() {
audit.CreatedAt = time.Now().UTC()
}
res, err := r.sess.Collection("workspace_file_audits").Insert(audit)
if err != nil {
return fmt.Errorf("failed to create workspace file audit: %w", err)
}
if id, ok := res.ID().(int64); ok {
audit.ID = id
}
return nil
}
backend/internal/services/instance_env.go
+18
View File
@@ -129,3 +129,21 @@ func buildInstancePodEnv(instance *models.Instance, runtimeEnv, gatewayEnv, agen
return resolved, nil
}
func buildInstanceGatewayEnv(instance *models.Instance, gatewayEnv map[string]string) (map[string]string, error) {
if instance == nil {
return nil, fmt.Errorf("instance is required")
}
overrides, err := parseEnvironmentOverridesJSON(instance.EnvironmentOverridesJSON)
if err != nil {
return nil, err
}
resolved := mergeEnvMaps(gatewayEnv, nil)
resolved = withInstanceProxyEnv(instance.Type, instance.ID, resolved)
resolved["CLAWMANAGER_RUNTIME_TYPE"] = normalizeInstanceRuntimeType(instance.RuntimeType)
resolved = mergeEnvMaps(resolved, overrides)
return resolved, nil
}
backend/internal/services/instance_external_access_service.go
+286
View File
@@ -0,0 +1,286 @@
package services
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
)
const (
ExternalAccessModeShareLink = "share_link"
ExternalAccessModePassword = "password"
ExternalAccessExpirationPreset = "preset"
ExternalAccessExpirationCustom = "custom"
ExternalAccessExpirationPermanent = "permanent"
ExternalAccessPreset1Hour = "1h"
ExternalAccessPreset24Hours = "24h"
ExternalAccessPreset7Days = "7d"
ExternalAccessPreset30Days = "30d"
)
type ExternalAccessExpirationRequest struct {
Mode string `json:"expires_mode,omitempty"`
Preset string `json:"expires_preset,omitempty"`
ExpiresAt *time.Time `json:"expires_at,omitempty"`
}
type EnableShareLinkResult struct {
Access *models.InstanceExternalAccess `json:"access"`
ShareURL string `json:"share_url,omitempty"`
}
type PasswordExternalAccessResult struct {
Access *models.InstanceExternalAccess `json:"access"`
Password string `json:"password"`
ShareURL string `json:"share_url,omitempty"`
}
type InstanceExternalAccessService interface {
Get(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error)
EnableShareLink(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*EnableShareLinkResult, error)
CreatePassword(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*PasswordExternalAccessResult, error)
Disable(ctx context.Context, instanceID int) error
ResolveShortLink(ctx context.Context, code string) (*models.InstanceExternalAccess, error)
ValidateShortLink(ctx context.Context, code, password string) (*models.InstanceExternalAccess, error)
}
type instanceExternalAccessService struct {
repo repository.InstanceExternalAccessRepository
}
func NewInstanceExternalAccessService(repo repository.InstanceExternalAccessRepository) InstanceExternalAccessService {
return &instanceExternalAccessService{repo: repo}
}
func (s *instanceExternalAccessService) Get(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) {
if s == nil || s.repo == nil {
return nil, fmt.Errorf("instance external access repository is not configured")
}
return s.repo.GetByInstanceID(ctx, instanceID)
}
func (s *instanceExternalAccessService) EnableShareLink(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*EnableShareLinkResult, error) {
if s == nil || s.repo == nil {
return nil, fmt.Errorf("instance external access repository is not configured")
}
code, codeHash, err := generateShortCode()
if err != nil {
return nil, err
}
expiresAt, err := resolveExternalAccessExpiresAt(expiration)
if err != nil {
return nil, err
}
access := &models.InstanceExternalAccess{
InstanceID: instanceID,
Enabled: true,
AuthMode: ExternalAccessModeShareLink,
ShortCodeHash: &codeHash,
PublicSlug: &code,
PublicTokenHash: nil,
ExpiresAt: expiresAt,
CreatedBy: &createdBy,
PasswordHash: nil,
PasswordValue: nil,
PasswordHint: nil,
LastUsedAt: nil,
}
if err := s.repo.Upsert(ctx, access); err != nil {
return nil, err
}
saved, err := s.repo.GetByInstanceID(ctx, instanceID)
if err != nil {
return nil, err
}
return &EnableShareLinkResult{
Access: saved,
ShareURL: shortExternalAccessURL(code),
}, nil
}
func (s *instanceExternalAccessService) CreatePassword(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*PasswordExternalAccessResult, error) {
if s == nil || s.repo == nil {
return nil, fmt.Errorf("instance external access repository is not configured")
}
code, codeHash, err := generateShortCode()
if err != nil {
return nil, err
}
password, err := randomToken("pwd", 16)
if err != nil {
return nil, err
}
expiresAt, err := resolveExternalAccessExpiresAt(expiration)
if err != nil {
return nil, err
}
passwordHash := hashExternalSecret(password)
hint := password
if len(hint) > 12 {
hint = hint[:12]
}
access := &models.InstanceExternalAccess{
InstanceID: instanceID,
Enabled: true,
AuthMode: ExternalAccessModePassword,
PublicSlug: &code,
ShortCodeHash: &codeHash,
PasswordHash: &passwordHash,
PasswordValue: &password,
PasswordHint: &hint,
ExpiresAt: expiresAt,
CreatedBy: &createdBy,
PublicTokenHash: nil,
LastUsedAt: nil,
}
if err := s.repo.Upsert(ctx, access); err != nil {
return nil, err
}
saved, err := s.repo.GetByInstanceID(ctx, instanceID)
if err != nil {
return nil, err
}
return &PasswordExternalAccessResult{Access: saved, Password: password, ShareURL: shortExternalAccessURL(code)}, nil
}
func (s *instanceExternalAccessService) Disable(ctx context.Context, instanceID int) error {
if s == nil || s.repo == nil {
return fmt.Errorf("instance external access repository is not configured")
}
return s.repo.Disable(ctx, instanceID)
}
func (s *instanceExternalAccessService) ResolveShortLink(ctx context.Context, code string) (*models.InstanceExternalAccess, error) {
if s == nil || s.repo == nil {
return nil, fmt.Errorf("instance external access repository is not configured")
}
code = strings.TrimSpace(code)
if code == "" {
return nil, fmt.Errorf("short link code is required")
}
access, err := s.repo.GetByShortCodeHash(ctx, hashExternalSecret(code))
if err != nil {
return nil, err
}
if access == nil || !access.Enabled {
return nil, fmt.Errorf("external access is not enabled")
}
if access.ExpiresAt != nil && time.Now().UTC().After(*access.ExpiresAt) {
return nil, fmt.Errorf("external access has expired")
}
return access, nil
}
func (s *instanceExternalAccessService) ValidateShortLink(ctx context.Context, code, password string) (*models.InstanceExternalAccess, error) {
access, err := s.ResolveShortLink(ctx, code)
if err != nil {
return nil, err
}
switch access.AuthMode {
case ExternalAccessModeShareLink:
// The short code itself is the share-link capability. A matching hash is enough.
case ExternalAccessModePassword:
if access.PasswordHash == nil || *access.PasswordHash != hashExternalSecret(strings.TrimSpace(password)) {
return nil, fmt.Errorf("invalid external access password")
}
default:
return nil, fmt.Errorf("unsupported external access mode")
}
if err := s.repo.MarkUsed(ctx, access.ID); err != nil {
return nil, err
}
return access, nil
}
func resolveExternalAccessExpiresAt(expiration ExternalAccessExpirationRequest) (*time.Time, error) {
mode := strings.TrimSpace(expiration.Mode)
if mode == "" {
mode = ExternalAccessExpirationPreset
}
switch mode {
case ExternalAccessExpirationPermanent:
return nil, nil
case ExternalAccessExpirationCustom:
if expiration.ExpiresAt == nil {
return nil, fmt.Errorf("custom external access expiration requires expires_at")
}
value := expiration.ExpiresAt.UTC()
return &value, nil
case ExternalAccessExpirationPreset:
preset := strings.TrimSpace(expiration.Preset)
if preset == "" {
preset = ExternalAccessPreset24Hours
}
duration, err := externalAccessPresetDuration(preset)
if err != nil {
return nil, err
}
value := time.Now().UTC().Add(duration)
return &value, nil
default:
return nil, fmt.Errorf("unsupported external access expiration mode")
}
}
func externalAccessPresetDuration(preset string) (time.Duration, error) {
switch strings.TrimSpace(preset) {
case ExternalAccessPreset1Hour:
return time.Hour, nil
case ExternalAccessPreset24Hours:
return 24 * time.Hour, nil
case ExternalAccessPreset7Days:
return 7 * 24 * time.Hour, nil
case ExternalAccessPreset30Days:
return 30 * 24 * time.Hour, nil
default:
return 0, fmt.Errorf("unsupported external access expiration preset")
}
}
func generateShortCode() (code, codeHash string, err error) {
code, err = randomToken("sl", 12)
if err != nil {
return "", "", err
}
return code, hashExternalSecret(code), nil
}
func shortExternalAccessURL(code string) string {
return fmt.Sprintf("/s/%s/", strings.Trim(strings.TrimSpace(code), "/"))
}
func ExternalAccessShareURL(access *models.InstanceExternalAccess) string {
if access == nil || !access.Enabled || access.PublicSlug == nil {
return ""
}
return shortExternalAccessURL(*access.PublicSlug)
}
func ExternalAccessPassword(access *models.InstanceExternalAccess) string {
if access == nil || !access.Enabled || access.AuthMode != ExternalAccessModePassword || access.PasswordValue == nil {
return ""
}
return *access.PasswordValue
}
func randomToken(prefix string, bytesLen int) (string, error) {
buf := make([]byte, bytesLen)
if _, err := rand.Read(buf); err != nil {
return "", fmt.Errorf("failed to generate token: %w", err)
}
return prefix + "_" + hex.EncodeToString(buf), nil
}
func hashExternalSecret(value string) string {
sum := sha256.Sum256([]byte(strings.TrimSpace(value)))
return hex.EncodeToString(sum[:])
}
backend/internal/services/instance_external_access_service_test.go
+304
View File
@@ -0,0 +1,304 @@
package services
import (
"context"
"strings"
"testing"
"time"
"clawreef/internal/models"
)
func TestInstanceExternalAccessShareLinkValidation(t *testing.T) {
repo := newFakeExternalAccessRepo()
service := NewInstanceExternalAccessService(repo)
result, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
if err != nil {
t.Fatalf("EnableShareLink failed: %v", err)
}
if !strings.HasPrefix(result.ShareURL, "/s/") || strings.Contains(result.ShareURL, "token=") || strings.Contains(result.ShareURL, "/api/v1/") || strings.Contains(result.ShareURL, "/proxy") {
t.Fatalf("unexpected short share URL: %q", result.ShareURL)
}
code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/")
if code == "" {
t.Fatalf("short code missing from URL %q", result.ShareURL)
}
if result.Access.ShortCodeHash == nil || *result.Access.ShortCodeHash == code {
t.Fatalf("expected stored short code to be hashed")
}
if result.Access.AuthMode != ExternalAccessModeShareLink {
t.Fatalf("auth mode = %q, want %q", result.Access.AuthMode, ExternalAccessModeShareLink)
}
access, err := service.ValidateShortLink(context.Background(), code, "")
if err != nil {
t.Fatalf("ValidateShortLink failed: %v", err)
}
if access.InstanceID != 42 {
t.Fatalf("validated instance ID = %d, want 42", access.InstanceID)
}
if repo.markUsedCount != 1 {
t.Fatalf("mark used count = %d, want 1", repo.markUsedCount)
}
if _, err := service.ValidateShortLink(context.Background(), "wrong-code", ""); err == nil {
t.Fatalf("expected wrong short code to fail")
}
}
func TestInstanceExternalAccessShareLinkPresetExpiration(t *testing.T) {
repo := newFakeExternalAccessRepo()
service := NewInstanceExternalAccessService(repo)
before := time.Now().UTC().Add(24 * time.Hour)
result, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{
Mode: ExternalAccessExpirationPreset,
Preset: ExternalAccessPreset24Hours,
})
if err != nil {
t.Fatalf("EnableShareLink failed: %v", err)
}
if result.Access.ExpiresAt == nil {
t.Fatal("preset expiration must set expires_at")
}
after := time.Now().UTC().Add(24 * time.Hour)
if result.Access.ExpiresAt.Before(before) || result.Access.ExpiresAt.After(after.Add(time.Second)) {
t.Fatalf("preset expires_at = %v, want around 24h from now", result.Access.ExpiresAt)
}
}
func TestInstanceExternalAccessShareLinkCustomAndPermanentExpiration(t *testing.T) {
repo := newFakeExternalAccessRepo()
service := NewInstanceExternalAccessService(repo)
customAt := time.Now().UTC().Add(3 * time.Hour).Truncate(time.Second)
custom, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{
Mode: ExternalAccessExpirationCustom,
ExpiresAt: &customAt,
})
if err != nil {
t.Fatalf("custom EnableShareLink failed: %v", err)
}
if custom.Access.ExpiresAt == nil || !custom.Access.ExpiresAt.Equal(customAt) {
t.Fatalf("custom expires_at = %v, want %v", custom.Access.ExpiresAt, customAt)
}
permanent, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
if err != nil {
t.Fatalf("permanent EnableShareLink failed: %v", err)
}
if permanent.Access.ExpiresAt != nil {
t.Fatalf("permanent expires_at = %v, want nil", permanent.Access.ExpiresAt)
}
}
func TestInstanceExternalAccessPasswordDisable(t *testing.T) {
repo := newFakeExternalAccessRepo()
service := NewInstanceExternalAccessService(repo)
result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
if err != nil {
t.Fatalf("CreatePassword failed: %v", err)
}
if !strings.HasPrefix(result.Password, "pwd_") {
t.Fatalf("unexpected password prefix: %q", result.Password)
}
if result.Access.PasswordHash == nil || *result.Access.PasswordHash == result.Password {
t.Fatalf("expected stored password to be hashed")
}
if result.Access.PasswordHint == nil || !strings.HasPrefix(result.Password, *result.Access.PasswordHint) {
t.Fatalf("stored password hint does not match generated password")
}
if result.Access.AuthMode != ExternalAccessModePassword {
t.Fatalf("auth mode = %q, want %q", result.Access.AuthMode, ExternalAccessModePassword)
}
if !strings.HasPrefix(result.ShareURL, "/s/") || strings.Contains(result.ShareURL, "token=") || strings.Contains(result.ShareURL, "/api/v1/") {
t.Fatalf("unexpected password short URL: %q", result.ShareURL)
}
code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/")
if _, err := service.ValidateShortLink(context.Background(), code, result.Password); err != nil {
t.Fatalf("ValidateShortLink with password failed: %v", err)
}
if err := service.Disable(context.Background(), 100); err != nil {
t.Fatalf("Disable failed: %v", err)
}
if _, err := service.ValidateShortLink(context.Background(), code, result.Password); err == nil {
t.Fatalf("expected disabled password access to fail")
}
}
func TestInstanceExternalAccessRestoresShareURLAfterRefresh(t *testing.T) {
repo := newFakeExternalAccessRepo()
service := NewInstanceExternalAccessService(repo)
result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
if err != nil {
t.Fatalf("CreatePassword failed: %v", err)
}
reloaded, err := service.Get(context.Background(), 100)
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if got := ExternalAccessShareURL(reloaded); got != result.ShareURL {
t.Fatalf("ExternalAccessShareURL after reload = %q, want %q", got, result.ShareURL)
}
}
func TestInstanceExternalAccessRestoresPasswordAfterRefresh(t *testing.T) {
repo := newFakeExternalAccessRepo()
service := NewInstanceExternalAccessService(repo)
result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
if err != nil {
t.Fatalf("CreatePassword failed: %v", err)
}
reloaded, err := service.Get(context.Background(), 100)
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if got := ExternalAccessPassword(reloaded); got != result.Password {
t.Fatalf("ExternalAccessPassword after reload = %q, want generated password", got)
}
}
func TestInstanceExternalAccessExpiration(t *testing.T) {
repo := newFakeExternalAccessRepo()
service := NewInstanceExternalAccessService(repo)
expiredAt := time.Now().UTC().Add(-time.Minute)
result, err := service.EnableShareLink(context.Background(), 77, 3, ExternalAccessExpirationRequest{
Mode: ExternalAccessExpirationCustom,
ExpiresAt: &expiredAt,
})
if err != nil {
t.Fatalf("EnableShareLink failed: %v", err)
}
code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/")
if _, err := service.ValidateShortLink(context.Background(), code, ""); err == nil {
t.Fatalf("expected expired public access to fail")
}
}
type fakeExternalAccessRepo struct {
byInstance map[int]*models.InstanceExternalAccess
byCodeHash map[string]*models.InstanceExternalAccess
nextID int64
markUsedCount int
}
func newFakeExternalAccessRepo() *fakeExternalAccessRepo {
return &fakeExternalAccessRepo{
byInstance: make(map[int]*models.InstanceExternalAccess),
byCodeHash: make(map[string]*models.InstanceExternalAccess),
nextID: 1,
}
}
func (r *fakeExternalAccessRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) {
_ = ctx
return cloneExternalAccess(r.byInstance[instanceID]), nil
}
func (r *fakeExternalAccessRepo) GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) {
_ = ctx
return cloneExternalAccess(r.byCodeHash[codeHash]), nil
}
func (r *fakeExternalAccessRepo) Upsert(ctx context.Context, access *models.InstanceExternalAccess) error {
_ = ctx
saved := cloneExternalAccess(access)
if saved.ID == 0 {
if existing := r.byInstance[saved.InstanceID]; existing != nil {
saved.ID = existing.ID
} else {
saved.ID = r.nextID
r.nextID++
}
}
if existing := r.byInstance[saved.InstanceID]; existing != nil && existing.ShortCodeHash != nil {
delete(r.byCodeHash, *existing.ShortCodeHash)
}
r.byInstance[saved.InstanceID] = cloneExternalAccess(saved)
if saved.ShortCodeHash != nil {
r.byCodeHash[*saved.ShortCodeHash] = cloneExternalAccess(saved)
}
return nil
}
func (r *fakeExternalAccessRepo) Disable(ctx context.Context, instanceID int) error {
_ = ctx
if existing := r.byInstance[instanceID]; existing != nil {
existing.Enabled = false
if existing.ShortCodeHash != nil {
if byCode := r.byCodeHash[*existing.ShortCodeHash]; byCode != nil {
byCode.Enabled = false
}
}
}
return nil
}
func (r *fakeExternalAccessRepo) MarkUsed(ctx context.Context, id int64) error {
_ = ctx
r.markUsedCount++
now := time.Now().UTC()
for _, access := range r.byInstance {
if access.ID == id {
access.LastUsedAt = &now
}
}
for _, access := range r.byCodeHash {
if access.ID == id {
access.LastUsedAt = &now
}
}
return nil
}
func cloneExternalAccess(access *models.InstanceExternalAccess) *models.InstanceExternalAccess {
if access == nil {
return nil
}
clone := *access
if access.PublicSlug != nil {
value := *access.PublicSlug
clone.PublicSlug = &value
}
if access.PublicTokenHash != nil {
value := *access.PublicTokenHash
clone.PublicTokenHash = &value
}
if access.ShortCodeHash != nil {
value := *access.ShortCodeHash
clone.ShortCodeHash = &value
}
if access.PasswordHash != nil {
value := *access.PasswordHash
clone.PasswordHash = &value
}
if access.PasswordValue != nil {
value := *access.PasswordValue
clone.PasswordValue = &value
}
if access.PasswordHint != nil {
value := *access.PasswordHint
clone.PasswordHint = &value
}
if access.ExpiresAt != nil {
value := *access.ExpiresAt
clone.ExpiresAt = &value
}
if access.CreatedBy != nil {
value := *access.CreatedBy
clone.CreatedBy = &value
}
if access.LastUsedAt != nil {
value := *access.LastUsedAt
clone.LastUsedAt = &value
}
return &clone
}
backend/internal/services/instance_proxy_service.go
+359 -43
View File
@@ -4,16 +4,20 @@ import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"sync"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services/k8s"
"github.com/gorilla/websocket"
@@ -21,14 +25,19 @@ import (
// InstanceProxyService handles proxying requests to instance pods
type InstanceProxyService struct {
serviceService *k8s.ServiceService
accessService *InstanceAccessService
httpClient *http.Client
serviceCache map[serviceCacheKey]serviceCacheEntry
serviceLookups map[serviceCacheKey]*serviceLookupCall
cacheMu sync.RWMutex
lookupMu sync.Mutex
serviceTTL time.Duration
serviceService *k8s.ServiceService
accessService *InstanceAccessService
instanceRepo repository.InstanceRepository
runtimePodRepo repository.RuntimePodRepository
bindingRepo repository.InstanceRuntimeBindingRepository
httpClient *http.Client
openClawGatewayToken string
openClawProxyOrigin string
serviceCache map[serviceCacheKey]serviceCacheEntry
serviceLookups map[serviceCacheKey]*serviceLookupCall
cacheMu sync.RWMutex
lookupMu sync.Mutex
serviceTTL time.Duration
}
type serviceCacheKey struct {
@@ -50,8 +59,20 @@ type serviceLookupCall struct {
const defaultServiceCacheTTL = 30 * time.Second
var ErrInstanceGatewayUnavailable = errors.New("instance gateway is not available")
type InstanceProxyServiceOption func(*InstanceProxyService)
func WithInstanceProxyRuntimeRepositories(instanceRepo repository.InstanceRepository, runtimePodRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository) InstanceProxyServiceOption {
return func(s *InstanceProxyService) {
s.instanceRepo = instanceRepo
s.runtimePodRepo = runtimePodRepo
s.bindingRepo = bindingRepo
}
}
// NewInstanceProxyService creates a new instance proxy service
func NewInstanceProxyService(accessService *InstanceAccessService) *InstanceProxyService {
func NewInstanceProxyService(accessService *InstanceAccessService, options ...InstanceProxyServiceOption) *InstanceProxyService {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@@ -63,7 +84,7 @@ func NewInstanceProxyService(accessService *InstanceAccessService) *InstanceProx
ForceAttemptHTTP2: true,
}
return &InstanceProxyService{
service := &InstanceProxyService{
serviceService: k8s.NewServiceService(),
accessService: accessService,
httpClient: &http.Client{
@@ -73,10 +94,18 @@ func NewInstanceProxyService(accessService *InstanceAccessService) *InstanceProx
return http.ErrUseLastResponse
},
},
serviceCache: make(map[serviceCacheKey]serviceCacheEntry),
serviceLookups: make(map[serviceCacheKey]*serviceLookupCall),
serviceTTL: defaultServiceCacheTTL,
openClawGatewayToken: strings.TrimSpace(os.Getenv("OPENCLAW_GATEWAY_TOKEN")),
openClawProxyOrigin: resolveOpenClawProxyOriginFromEnv(),
serviceCache: make(map[serviceCacheKey]serviceCacheEntry),
serviceLookups: make(map[serviceCacheKey]*serviceLookupCall),
serviceTTL: defaultServiceCacheTTL,
}
for _, option := range options {
if option != nil {
option(service)
}
}
return service
}
// ProxyRequest proxies a request to an instance
@@ -102,27 +131,22 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
return fmt.Errorf("token does not match instance")
}
// Extract the actual path from the request (remove the proxy prefix)
targetPath := s.extractTargetPath(r.URL.Path, instanceID, accessToken.InstanceType)
targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath)
shouldRewriteHTML := s.shouldRewriteHTML(accessToken.InstanceType)
effectiveRequestPath := canonicalProxyEntryRequestPath(r.URL.Path, accessToken, instanceID)
// Get service info for the instance (create if not exists)
serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort)
if err != nil {
return fmt.Errorf("failed to get or create service: %w", err)
}
// Extract the actual path from the request (remove the proxy prefix)
targetPath := s.extractTargetPath(effectiveRequestPath, instanceID, accessToken.InstanceType)
targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath)
shouldRewriteHTML := s.shouldRewriteHTMLForProxy(instanceID, accessToken.InstanceType)
// Build target URL
targetURL := &url.URL{
Scheme: s.resolveTargetScheme(accessToken.InstanceType, false),
Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo),
Path: targetPath,
targetURL, err := s.resolveHTTPProxyTarget(ctx, accessToken, instanceID, targetPort, targetPath, effectiveRequestPath)
if err != nil {
return err
}
// Copy query parameters (excluding token)
queryParams := r.URL.Query()
queryParams.Del("token")
removeProxyAccessTokenQuery(queryParams, token)
if len(queryParams) > 0 {
targetURL.RawQuery = queryParams.Encode()
}
@@ -148,6 +172,9 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
proxyReq.Header.Set("X-Forwarded-Host", r.Host)
proxyReq.Header.Set("X-Forwarded-Proto", requestScheme(r))
proxyReq.Header.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID))
if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" {
proxyReq.Header.Set("Authorization", "Bearer "+token)
}
if shouldRewriteHTML {
proxyReq.Header.Del("Accept-Encoding")
}
@@ -179,7 +206,7 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
return fmt.Errorf("failed to close upstream html body: %w", closeErr)
}
modifiedBody := injectProxyBase(string(body), fmt.Sprintf("/api/v1/instances/%d/proxy/", instanceID))
modifiedBody := injectProxyBase(string(body), proxyBaseForRequestPath(effectiveRequestPath, instanceID))
resp.Body = io.NopCloser(bytes.NewReader([]byte(modifiedBody)))
resp.ContentLength = int64(len(modifiedBody))
resp.Header.Set("Content-Length", strconv.Itoa(len(modifiedBody)))
@@ -238,22 +265,14 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in
targetPath := s.extractTargetPath(r.URL.Path, instanceID, accessToken.InstanceType)
targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath)
// Get service info for the instance
serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort)
targetURL, err := s.resolveWebSocketProxyTarget(ctx, accessToken, instanceID, targetPort, targetPath, r.URL.Path)
if err != nil {
return fmt.Errorf("failed to get or create service: %w", err)
}
// WebSocket upstream uses ws/wss explicitly.
targetURL := &url.URL{
Scheme: s.resolveTargetScheme(accessToken.InstanceType, true),
Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo),
Path: targetPath,
return err
}
// Copy query parameters (excluding token)
queryParams := r.URL.Query()
queryParams.Del("token")
removeProxyAccessTokenQuery(queryParams, token)
if len(queryParams) > 0 {
targetURL.RawQuery = queryParams.Encode()
}
@@ -270,8 +289,14 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in
upstreamHeader.Del("Sec-Websocket-Key")
upstreamHeader.Del("Sec-Websocket-Version")
upstreamHeader.Del("Sec-Websocket-Extensions")
upstreamHeader.Set("X-Forwarded-For", r.RemoteAddr)
upstreamHeader.Set("X-Forwarded-Host", r.Host)
upstreamHeader.Set("X-Forwarded-Proto", requestScheme(r))
upstreamHeader.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID))
if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" {
upstreamHeader.Set("Authorization", "Bearer "+token)
upstreamHeader.Set("Origin", s.openClawWebSocketOrigin(targetURL))
}
dialer := websocket.Dialer{
Proxy: http.ProxyFromEnvironment,
@@ -364,6 +389,112 @@ func (s *InstanceProxyService) removeHopByHopHeaders(header http.Header) {
}
}
func (s *InstanceProxyService) managedRuntimeGatewayBearerToken(ctx context.Context, instanceID int, instanceType string) string {
if s == nil || s.instanceRepo == nil {
return ""
}
normalizedType, managedType := NormalizeV2RuntimeType(instanceType)
if !managedType {
return ""
}
instance, err := s.instanceRepo.GetByID(instanceID)
if err != nil || instance == nil {
return ""
}
if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == normalizedType {
if instance.AccessToken != nil && strings.TrimSpace(*instance.AccessToken) != "" {
return strings.TrimSpace(*instance.AccessToken)
}
}
if normalizedType == RuntimeTypeOpenClaw && s.openClawGatewayToken != "" {
return s.openClawGatewayToken
}
return ""
}
func (s *InstanceProxyService) resolveHTTPProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPort int32, targetPath, requestPath string) (*url.URL, error) {
if targetURL, ok, err := s.resolveV2ProxyTarget(ctx, accessToken, instanceID, targetPath, requestPath, false); ok || err != nil {
return targetURL, err
}
serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort)
if err != nil {
return nil, fmt.Errorf("failed to get or create service: %w", err)
}
return &url.URL{
Scheme: s.resolveTargetScheme(accessToken.InstanceType, false),
Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo),
Path: targetPath,
}, nil
}
func (s *InstanceProxyService) resolveWebSocketProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPort int32, targetPath, requestPath string) (*url.URL, error) {
if targetURL, ok, err := s.resolveV2ProxyTarget(ctx, accessToken, instanceID, targetPath, requestPath, true); ok || err != nil {
return targetURL, err
}
serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort)
if err != nil {
return nil, fmt.Errorf("failed to get or create service: %w", err)
}
return &url.URL{
Scheme: s.resolveTargetScheme(accessToken.InstanceType, true),
Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo),
Path: targetPath,
}, nil
}
func (s *InstanceProxyService) resolveV2ProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPath, requestPath string, websocket bool) (*url.URL, bool, error) {
if s.instanceRepo == nil || s.bindingRepo == nil || s.runtimePodRepo == nil {
return nil, false, nil
}
instance, err := s.instanceRepo.GetByID(instanceID)
if err != nil {
return nil, false, fmt.Errorf("failed to get instance for proxy: %w", err)
}
if instance == nil {
return nil, false, ErrInstanceGatewayUnavailable
}
if instance.UserID != accessToken.UserID {
return nil, false, fmt.Errorf("token does not match instance owner")
}
if _, ok := v2RuntimeTypeForInstance(instance); !ok {
return nil, false, nil
}
if !strings.EqualFold(strings.TrimSpace(instance.Status), "running") {
return nil, true, ErrInstanceGatewayUnavailable
}
binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID)
if err != nil {
return nil, true, fmt.Errorf("%w: %v", ErrInstanceGatewayUnavailable, err)
}
if binding == nil {
return nil, true, ErrInstanceGatewayUnavailable
}
if binding.Generation != instance.RuntimeGeneration {
return nil, true, ErrInstanceGatewayUnavailable
}
pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
if err != nil {
return nil, true, fmt.Errorf("%w: %v", ErrInstanceGatewayUnavailable, err)
}
if pod == nil || pod.PodIP == nil || strings.TrimSpace(*pod.PodIP) == "" || binding.GatewayPort <= 0 {
return nil, true, ErrInstanceGatewayUnavailable
}
scheme := "http"
if websocket {
scheme = "ws"
}
upstreamPath := stripInstanceProxyPrefix(targetPath, instanceID)
if shouldPreserveOpenClawControlUIPath(instance) {
upstreamPath = openClawControlUIRequestPath(requestPath, instanceID)
}
return &url.URL{
Scheme: scheme,
Host: net.JoinHostPort(strings.TrimSpace(*pod.PodIP), strconv.Itoa(binding.GatewayPort)),
Path: upstreamPath,
}, true, nil
}
// getOrCreateService gets service info or creates the service if it doesn't exist
func (s *InstanceProxyService) getOrCreateService(ctx context.Context, userID, instanceID int, targetPort int32) (*k8s.ServiceInfo, error) {
cacheKey := serviceCacheKey{
@@ -445,13 +576,78 @@ func (s *InstanceProxyService) extractTargetPath(requestPath string, instanceID
return requestPath
}
func stripInstanceProxyPrefix(requestPath string, instanceID int) string {
prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
if strings.HasPrefix(requestPath, prefix) {
path := strings.TrimPrefix(requestPath, prefix)
if path == "" {
return "/"
}
return path
}
return requestPath
}
func canonicalProxyEntryRequestPath(requestPath string, accessToken *AccessToken, instanceID int) string {
prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
path := strings.TrimSpace(requestPath)
if path != prefix && path != prefix+"/" {
return requestPath
}
if accessToken == nil || strings.TrimSpace(accessToken.AccessURL) == "" {
return requestPath
}
parsed, err := url.Parse(accessToken.AccessURL)
if err != nil {
return requestPath
}
entryPath := strings.TrimSpace(parsed.Path)
if entryPath == "" || entryPath == prefix || entryPath == prefix+"/" {
return requestPath
}
if strings.HasPrefix(entryPath, prefix+"/") {
return entryPath
}
return requestPath
}
func shouldPreserveOpenClawControlUIPath(instance *models.Instance) bool {
if instance == nil || !strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeOpenClaw) {
return false
}
_, ok := v2RuntimeTypeForInstance(instance)
return ok
}
func openClawControlUIRequestPath(requestPath string, instanceID int) string {
prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
path := strings.TrimSpace(requestPath)
if path == "" || path == prefix {
return prefix + "/"
}
if strings.HasPrefix(path, prefix) {
return path
}
if strings.HasPrefix(path, "/") {
return prefix + path
}
return prefix + "/" + path
}
// GetProxyURL generates a proxy URL for frontend
func (s *InstanceProxyService) GetProxyURL(instanceID int, token string) string {
if token == "" {
return fmt.Sprintf("/api/v1/instances/%d/proxy/", instanceID)
}
return proxyURLWithPath(instanceID, "/", token)
}
return fmt.Sprintf("/api/v1/instances/%d/proxy/?token=%s", instanceID, token)
// GetProxyURLForInstance generates the best frontend entry URL for an instance.
func (s *InstanceProxyService) GetProxyURLForInstance(instance *models.Instance, token string) string {
if instance == nil {
return ""
}
if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == RuntimeTypeHermes {
return proxyURLWithPath(instance.ID, "/chat", token)
}
return proxyURLWithPath(instance.ID, "/", token)
}
// GetTargetPortForInstance returns the service target port used by the instance type.
@@ -526,6 +722,18 @@ func (s *InstanceProxyService) shouldRewriteHTML(instanceType string) bool {
return !usesWebtopImage(instanceType)
}
func (s *InstanceProxyService) shouldRewriteHTMLForProxy(instanceID int, instanceType string) bool {
if s != nil && s.instanceRepo != nil && strings.EqualFold(strings.TrimSpace(instanceType), RuntimeTypeHermes) {
instance, err := s.instanceRepo.GetByID(instanceID)
if err == nil && instance != nil {
if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == RuntimeTypeHermes {
return true
}
}
}
return s.shouldRewriteHTML(instanceType)
}
func (s *InstanceProxyService) getCachedService(key serviceCacheKey) *k8s.ServiceInfo {
s.cacheMu.RLock()
entry, ok := s.serviceCache[key]
@@ -593,6 +801,114 @@ func injectProxyBase(html, proxyBase string) string {
return baseTag + html
}
func proxyURLWithPath(instanceID int, targetPath, token string) string {
path := strings.TrimSpace(targetPath)
if path == "" || path == "/" {
path = "/"
} else {
path = "/" + strings.TrimLeft(path, "/")
if !strings.HasSuffix(path, "/") {
path += "/"
}
}
raw := fmt.Sprintf("/api/v1/instances/%d/proxy%s", instanceID, path)
if token == "" {
return raw
}
return fmt.Sprintf("%s?token=%s", raw, url.QueryEscape(token))
}
func removeProxyAccessTokenQuery(query url.Values, accessToken string) {
values := query["token"]
if len(values) == 0 {
return
}
filtered := values[:0]
for _, value := range values {
if value != accessToken {
filtered = append(filtered, value)
}
}
if len(filtered) == 0 {
query.Del("token")
return
}
query["token"] = filtered
}
func proxyBaseForRequestPath(requestPath string, instanceID int) string {
prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
path := strings.TrimSpace(requestPath)
if strings.HasPrefix(path, prefix) {
path = strings.TrimPrefix(path, prefix)
}
if path == "" || path == "/" {
return fmt.Sprintf("%s/", prefix)
}
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
if !strings.HasSuffix(path, "/") {
lastSlash := strings.LastIndex(path, "/")
if lastSlash >= 0 {
path = path[:lastSlash+1]
} else {
path = "/"
}
}
return prefix + path
}
func websocketUpstreamOrigin(targetURL *url.URL) string {
if targetURL == nil {
return ""
}
scheme := targetURL.Scheme
switch scheme {
case "ws":
scheme = "http"
case "wss":
scheme = "https"
}
if scheme == "" || targetURL.Host == "" {
return ""
}
return scheme + "://" + targetURL.Host
}
func (s *InstanceProxyService) openClawWebSocketOrigin(targetURL *url.URL) string {
if s != nil && s.openClawProxyOrigin != "" {
return s.openClawProxyOrigin
}
return websocketUpstreamOrigin(targetURL)
}
func resolveOpenClawProxyOriginFromEnv() string {
for _, key := range []string{
"OPENCLAW_PROXY_ORIGIN",
"CLAWMANAGER_TEAM_MANAGER_BASE_URL",
"CLAWMANAGER_BACKEND_URL",
} {
if origin := originFromURLString(os.Getenv(key)); origin != "" {
return origin
}
}
return ""
}
func originFromURLString(rawURL string) string {
value := strings.TrimSpace(rawURL)
if value == "" {
return ""
}
parsed, err := url.Parse(value)
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
return ""
}
return parsed.Scheme + "://" + parsed.Host
}
func (s *InstanceProxyService) rewriteRedirectLocation(instanceID int, location string) string {
if strings.HasPrefix(location, "/") && !strings.HasPrefix(location, "/api/v1/instances/") {
return fmt.Sprintf("/api/v1/instances/%d/proxy%s", instanceID, location)
backend/internal/services/instance_proxy_service_v2_test.go
+692
View File
@@ -0,0 +1,692 @@
package services
import (
"errors"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
"github.com/gorilla/websocket"
)
func TestInstanceProxyServiceUsesRuntimeBindingForV2(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/instances/123/proxy/apps/openclaw" {
t.Fatalf("unexpected upstream path %s", r.URL.Path)
}
if got := r.Header.Get("X-Forwarded-Prefix"); got != "/api/v1/instances/123/proxy" {
t.Fatalf("X-Forwarded-Prefix = %q", got)
}
if got := r.Header.Get("Authorization"); got != "Bearer internal-openclaw-token" {
t.Fatalf("Authorization = %q", got)
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
}))
defer upstream.Close()
podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
workspacePath := "/workspaces/openclaw/user-45/instance-123"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[123] = &models.Instance{
ID: 123,
UserID: 45,
Type: "openclaw",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
RuntimeGeneration: 3,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[123] = &models.InstanceRuntimeBinding{
InstanceID: 123,
RuntimePodID: 9,
GatewayPort: gatewayPort,
State: "running",
Generation: 3,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
9: {ID: 9, PodIP: &podIP, State: "ready"},
},
}
accessService := NewInstanceAccessService()
defer accessService.Stop()
token, err := accessService.GenerateToken(45, 123, "openclaw", "/api/v1/instances/123/proxy/", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
service.instanceRepo = instanceRepo
service.bindingRepo = bindingRepo
service.runtimePodRepo = podRepo
service.httpClient = upstream.Client()
service.openClawGatewayToken = "internal-openclaw-token"
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/123/proxy/apps/openclaw?token="+url.QueryEscape(token.Token), nil)
rec := httptest.NewRecorder()
if err := service.ProxyRequest(req.Context(), 123, token.Token, rec, req); err != nil {
t.Fatalf("ProxyRequest returned error: %v", err)
}
if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
}
}
func TestInstanceProxyServiceInjectsInstanceTokenForHermesLite(t *testing.T) {
instanceToken := "igt_hermes_instance"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/chat" {
t.Fatalf("unexpected upstream path %s", r.URL.Path)
}
if r.URL.RawQuery != "" {
t.Fatalf("unexpected upstream query %q", r.URL.RawQuery)
}
if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
t.Fatalf("Authorization = %q", got)
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
}))
defer upstream.Close()
podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
workspacePath := "/workspaces/hermes/user-45/instance-127"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[127] = &models.Instance{
ID: 127,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
InstanceMode: InstanceModeLite,
Status: "running",
AccessToken: &instanceToken,
WorkspacePath: &workspacePath,
RuntimeGeneration: 5,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[127] = &models.InstanceRuntimeBinding{
InstanceID: 127,
RuntimePodID: 10,
GatewayPort: gatewayPort,
State: "running",
Generation: 5,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
10: {ID: 10, PodIP: &podIP, State: "ready"},
},
}
accessService := NewInstanceAccessService()
defer accessService.Stop()
token, err := accessService.GenerateToken(45, 127, "hermes", "/api/v1/instances/127/proxy/", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
service.instanceRepo = instanceRepo
service.bindingRepo = bindingRepo
service.runtimePodRepo = podRepo
service.httpClient = upstream.Client()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/127/proxy/chat?token="+url.QueryEscape(token.Token), nil)
rec := httptest.NewRecorder()
if err := service.ProxyRequest(req.Context(), 127, token.Token, rec, req); err != nil {
t.Fatalf("ProxyRequest returned error: %v", err)
}
if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
}
}
func TestInstanceProxyServiceUsesHermesLiteAccessURLForRootEntry(t *testing.T) {
instanceToken := "igt_hermes_instance"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/chat/" {
t.Fatalf("unexpected upstream path %s", r.URL.Path)
}
if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
t.Fatalf("Authorization = %q", got)
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("<!doctype html><html><head><title>Hermes</title></head><body>ok</body></html>"))
}))
defer upstream.Close()
podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
workspacePath := "/workspaces/hermes/user-45/instance-131"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[131] = &models.Instance{
ID: 131,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
InstanceMode: InstanceModeLite,
Status: "running",
AccessToken: &instanceToken,
WorkspacePath: &workspacePath,
RuntimeGeneration: 5,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[131] = &models.InstanceRuntimeBinding{
InstanceID: 131,
RuntimePodID: 14,
GatewayPort: gatewayPort,
State: "running",
Generation: 5,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
14: {ID: 14, PodIP: &podIP, State: "ready"},
},
}
accessService := NewInstanceAccessService()
defer accessService.Stop()
token, err := accessService.GenerateToken(45, 131, "hermes", "/api/v1/instances/131/proxy/chat/", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
service.instanceRepo = instanceRepo
service.bindingRepo = bindingRepo
service.runtimePodRepo = podRepo
service.httpClient = upstream.Client()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/131/proxy/?token="+url.QueryEscape(token.Token), nil)
rec := httptest.NewRecorder()
if err := service.ProxyRequest(req.Context(), 131, token.Token, rec, req); err != nil {
t.Fatalf("ProxyRequest returned error: %v", err)
}
if rec.Code != http.StatusOK {
t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), `<base href="/api/v1/instances/131/proxy/chat/">`) {
t.Fatalf("expected Hermes Lite HTML to include chat proxy base, got %q", rec.Body.String())
}
}
func TestInstanceProxyServicePreservesHermesRuntimeQueryToken(t *testing.T) {
instanceToken := "igt_hermes_instance"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/ws" {
t.Fatalf("unexpected upstream path %s", r.URL.Path)
}
if got := r.URL.Query().Get("token"); got != "hermes-session-token" {
t.Fatalf("upstream token query = %q", got)
}
if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
t.Fatalf("Authorization = %q", got)
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
}))
defer upstream.Close()
podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
workspacePath := "/workspaces/hermes/user-45/instance-130"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[130] = &models.Instance{
ID: 130,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
InstanceMode: InstanceModeLite,
Status: "running",
AccessToken: &instanceToken,
WorkspacePath: &workspacePath,
RuntimeGeneration: 5,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[130] = &models.InstanceRuntimeBinding{
InstanceID: 130,
RuntimePodID: 13,
GatewayPort: gatewayPort,
State: "running",
Generation: 5,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
13: {ID: 13, PodIP: &podIP, State: "ready"},
},
}
accessService := NewInstanceAccessService()
defer accessService.Stop()
token, err := accessService.GenerateToken(45, 130, "hermes", "/api/v1/instances/130/proxy/chat/", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
service.instanceRepo = instanceRepo
service.bindingRepo = bindingRepo
service.runtimePodRepo = podRepo
service.httpClient = upstream.Client()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/130/proxy/api/ws?token=hermes-session-token", nil)
rec := httptest.NewRecorder()
if err := service.ProxyRequest(req.Context(), 130, token.Token, rec, req); err != nil {
t.Fatalf("ProxyRequest returned error: %v", err)
}
if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
}
}
func TestInstanceProxyServiceRewritesHermesLiteHTMLBase(t *testing.T) {
instanceToken := "igt_hermes_instance"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
t.Fatalf("unexpected upstream path %s", r.URL.Path)
}
if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
t.Fatalf("Authorization = %q", got)
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("<!doctype html><html><head><title>Hermes</title></head><body>ok</body></html>"))
}))
defer upstream.Close()
podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
workspacePath := "/workspaces/hermes/user-45/instance-128"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[128] = &models.Instance{
ID: 128,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
InstanceMode: InstanceModeLite,
Status: "running",
AccessToken: &instanceToken,
WorkspacePath: &workspacePath,
RuntimeGeneration: 5,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[128] = &models.InstanceRuntimeBinding{
InstanceID: 128,
RuntimePodID: 11,
GatewayPort: gatewayPort,
State: "running",
Generation: 5,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
11: {ID: 11, PodIP: &podIP, State: "ready"},
},
}
accessService := NewInstanceAccessService()
defer accessService.Stop()
token, err := accessService.GenerateToken(45, 128, "hermes", "/api/v1/instances/128/proxy/", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
service.instanceRepo = instanceRepo
service.bindingRepo = bindingRepo
service.runtimePodRepo = podRepo
service.httpClient = upstream.Client()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/128/proxy/?token="+url.QueryEscape(token.Token), nil)
rec := httptest.NewRecorder()
if err := service.ProxyRequest(req.Context(), 128, token.Token, rec, req); err != nil {
t.Fatalf("ProxyRequest returned error: %v", err)
}
if rec.Code != http.StatusOK {
t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), `<base href="/api/v1/instances/128/proxy/">`) {
t.Fatalf("expected Hermes Lite HTML to include proxy base, got %q", rec.Body.String())
}
}
func TestInstanceProxyServiceProxiesHermesLiteWebSocket(t *testing.T) {
instanceToken := "igt_hermes_instance"
upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/ws" {
t.Fatalf("unexpected upstream websocket path %s", r.URL.Path)
}
if got := r.URL.Query().Get("token"); got != "hermes-ws-token" {
t.Fatalf("upstream websocket token query = %q", got)
}
if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
t.Fatalf("Authorization = %q", got)
}
if got := r.Header.Get("X-Forwarded-Prefix"); got != "/api/v1/instances/129/proxy" {
t.Fatalf("X-Forwarded-Prefix = %q", got)
}
if got := r.Header.Get("X-Forwarded-Host"); got == "" {
t.Fatalf("X-Forwarded-Host is empty")
}
if got := r.Header.Get("X-Forwarded-For"); got == "" {
t.Fatalf("X-Forwarded-For is empty")
}
if got, want := r.Header.Get("Origin"), "http://"+r.Host; got != want {
t.Fatalf("Origin = %q, want %q", got, want)
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
t.Fatalf("upstream websocket upgrade failed: %v", err)
}
defer conn.Close()
messageType, message, err := conn.ReadMessage()
if err != nil {
t.Fatalf("upstream websocket read failed: %v", err)
}
if string(message) != "ping" {
t.Fatalf("upstream websocket message = %q", message)
}
if err := conn.WriteMessage(messageType, []byte("pong")); err != nil {
t.Fatalf("upstream websocket write failed: %v", err)
}
}))
defer upstream.Close()
podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
workspacePath := "/workspaces/hermes/user-45/instance-129"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[129] = &models.Instance{
ID: 129,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
InstanceMode: InstanceModeLite,
Status: "running",
AccessToken: &instanceToken,
WorkspacePath: &workspacePath,
RuntimeGeneration: 5,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[129] = &models.InstanceRuntimeBinding{
InstanceID: 129,
RuntimePodID: 12,
GatewayPort: gatewayPort,
State: "running",
Generation: 5,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
12: {ID: 12, PodIP: &podIP, State: "ready"},
},
}
accessService := NewInstanceAccessService()
defer accessService.Stop()
token, err := accessService.GenerateToken(45, 129, "hermes", "/api/v1/instances/129/proxy/", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
service.instanceRepo = instanceRepo
service.bindingRepo = bindingRepo
service.runtimePodRepo = podRepo
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := service.ProxyWebSocket(r.Context(), 129, token.Token, w, r); err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
}
}))
defer proxy.Close()
wsURL := "ws" + strings.TrimPrefix(proxy.URL, "http") + "/api/v1/instances/129/proxy/ws?token=hermes-ws-token"
clientConn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
if err != nil {
t.Fatalf("client websocket dial failed: %v", err)
}
defer clientConn.Close()
if err := clientConn.WriteMessage(websocket.TextMessage, []byte("ping")); err != nil {
t.Fatalf("client websocket write failed: %v", err)
}
_, message, err := clientConn.ReadMessage()
if err != nil {
t.Fatalf("client websocket read failed: %v", err)
}
if string(message) != "pong" {
t.Fatalf("client websocket message = %q", message)
}
}
func TestInstanceProxyServiceUsesBaseProxyEntryForV2OpenClaw(t *testing.T) {
workspacePath := "/workspaces/openclaw/user-45/instance-123"
accessService := NewInstanceAccessService()
t.Cleanup(accessService.Stop)
service := NewInstanceProxyService(accessService)
got := service.GetProxyURLForInstance(&models.Instance{
ID: 123,
Type: "openclaw",
RuntimeType: "gateway",
WorkspacePath: &workspacePath,
}, "token+with/slash")
want := "/api/v1/instances/123/proxy/?token=token%2Bwith%2Fslash"
if got != want {
t.Fatalf("GetProxyURLForInstance() = %q, want %q", got, want)
}
}
func TestInstanceProxyServiceUsesChatEntryForHermesLite(t *testing.T) {
workspacePath := "/workspaces/hermes/user-45/instance-123"
accessService := NewInstanceAccessService()
t.Cleanup(accessService.Stop)
service := NewInstanceProxyService(accessService)
got := service.GetProxyURLForInstance(&models.Instance{
ID: 123,
Type: "hermes",
RuntimeType: "gateway",
InstanceMode: InstanceModeLite,
WorkspacePath: &workspacePath,
}, "token+with/slash")
want := "/api/v1/instances/123/proxy/chat/?token=token%2Bwith%2Fslash"
if got != want {
t.Fatalf("GetProxyURLForInstance() = %q, want %q", got, want)
}
}
func TestProxyBaseForRequestPathUsesSubdirectory(t *testing.T) {
got := proxyBaseForRequestPath("/api/v1/instances/123/proxy/__openclaw__/canvas/", 123)
want := "/api/v1/instances/123/proxy/__openclaw__/canvas/"
if got != want {
t.Fatalf("proxyBaseForRequestPath() = %q, want %q", got, want)
}
got = proxyBaseForRequestPath("/api/v1/instances/123/proxy/__openclaw__/canvas/app.js", 123)
if got != want {
t.Fatalf("proxyBaseForRequestPath(file) = %q, want %q", got, want)
}
}
func TestResolveOpenClawProxyOriginFromEnvUsesInternalOrigin(t *testing.T) {
t.Setenv("OPENCLAW_PROXY_ORIGIN", "")
t.Setenv("CLAWMANAGER_BACKEND_URL", "")
t.Setenv("CLAWMANAGER_TEAM_MANAGER_BASE_URL", "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api")
got := resolveOpenClawProxyOriginFromEnv()
want := "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
if got != want {
t.Fatalf("resolveOpenClawProxyOriginFromEnv() = %q, want %q", got, want)
}
}
func TestOpenClawWebSocketOriginPrefersConfiguredProxyOrigin(t *testing.T) {
accessService := NewInstanceAccessService()
t.Cleanup(accessService.Stop)
service := NewInstanceProxyService(accessService)
service.openClawProxyOrigin = "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
got := service.openClawWebSocketOrigin(&url.URL{Scheme: "ws", Host: "10.42.0.63:20000"})
want := "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
if got != want {
t.Fatalf("openClawWebSocketOrigin() = %q, want %q", got, want)
}
}
func TestWebSocketUpstreamOriginFallsBackToGatewayHost(t *testing.T) {
tests := []struct {
name string
url *url.URL
want string
}{
{
name: "plain websocket",
url: &url.URL{Scheme: "ws", Host: "10.42.0.63:20000"},
want: "http://10.42.0.63:20000",
},
{
name: "tls websocket",
url: &url.URL{Scheme: "wss", Host: "gateway.example.test"},
want: "https://gateway.example.test",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := websocketUpstreamOrigin(tt.url); got != tt.want {
t.Fatalf("websocketUpstreamOrigin() = %q, want %q", got, tt.want)
}
})
}
}
func TestInstanceProxyServiceRejectsStoppedV2InstanceWithStaleBinding(t *testing.T) {
workspacePath := "/workspaces/openclaw/user-45/instance-125"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[125] = &models.Instance{
ID: 125,
UserID: 45,
Type: "openclaw",
RuntimeType: "gateway",
Status: "stopped",
WorkspacePath: &workspacePath,
RuntimeGeneration: 4,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[125] = &models.InstanceRuntimeBinding{
InstanceID: 125,
RuntimePodID: 9,
GatewayPort: 20025,
State: "running",
Generation: 4,
}
podIP := "10.42.0.125"
service, token := newV2ProxyTestService(t, instanceRepo, bindingRepo, &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
9: {ID: 9, PodIP: &podIP},
},
}, 45, 125, "openclaw")
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/125/proxy/", nil)
err := service.ProxyRequest(req.Context(), 125, token.Token, httptest.NewRecorder(), req)
if !errors.Is(err, ErrInstanceGatewayUnavailable) {
t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err)
}
}
func TestInstanceProxyServiceRejectsV2BindingGenerationMismatch(t *testing.T) {
workspacePath := "/workspaces/hermes/user-45/instance-126"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[126] = &models.Instance{
ID: 126,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
RuntimeGeneration: 8,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[126] = &models.InstanceRuntimeBinding{
InstanceID: 126,
RuntimePodID: 10,
GatewayPort: 20026,
State: "running",
Generation: 7,
}
podIP := "10.42.0.126"
service, token := newV2ProxyTestService(t, instanceRepo, bindingRepo, &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
10: {ID: 10, PodIP: &podIP},
},
}, 45, 126, "hermes")
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/126/proxy/", nil)
err := service.ProxyRequest(req.Context(), 126, token.Token, httptest.NewRecorder(), req)
if !errors.Is(err, ErrInstanceGatewayUnavailable) {
t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err)
}
}
func TestInstanceProxyServiceReturnsUnavailableWhenV2BindingMissing(t *testing.T) {
workspacePath := "/workspaces/hermes/user-45/instance-124"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[124] = &models.Instance{
ID: 124,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
}
accessService := NewInstanceAccessService()
defer accessService.Stop()
token, err := accessService.GenerateToken(45, 124, "hermes", "/api/v1/instances/124/proxy/", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
service.instanceRepo = instanceRepo
service.bindingRepo = newFakeRuntimeBindingRepo()
service.runtimePodRepo = &fakeRuntimePodRepo{}
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/124/proxy/", nil)
rec := httptest.NewRecorder()
err = service.ProxyRequest(req.Context(), 124, token.Token, rec, req)
if !errors.Is(err, ErrInstanceGatewayUnavailable) {
t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err)
}
}
func newV2ProxyTestService(t *testing.T, instanceRepo repository.InstanceRepository, bindingRepo repository.InstanceRuntimeBindingRepository, podRepo repository.RuntimePodRepository, userID int, instanceID int, instanceType string) (*InstanceProxyService, *AccessToken) {
t.Helper()
accessService := NewInstanceAccessService()
t.Cleanup(accessService.Stop)
token, err := accessService.GenerateToken(userID, instanceID, instanceType, "/api/v1/instances/"+strconv.Itoa(instanceID)+"/proxy/", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
service.instanceRepo = instanceRepo
service.bindingRepo = bindingRepo
service.runtimePodRepo = podRepo
return service, token
}
func splitURLHostPortForProxyTest(t *testing.T, rawURL string) (string, int) {
t.Helper()
parsed, err := url.Parse(rawURL)
if err != nil {
t.Fatalf("parse upstream URL: %v", err)
}
host, portText, err := net.SplitHostPort(parsed.Host)
if err != nil {
t.Fatalf("split upstream host port: %v", err)
}
port, err := strconv.Atoi(portText)
if err != nil {
t.Fatalf("parse upstream port: %v", err)
}
return host, port
}
backend/internal/services/instance_service.go
+643 -74
View File
@@ -6,6 +6,8 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path"
"strconv"
"strings"
"time"
@@ -79,11 +81,13 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn
currentGPU := 0
existingNames := map[string]struct{}{}
for _, existing := range existingInstances {
currentCPU += existing.CPUCores
currentMemory += existing.MemoryGB
currentStorage += existing.DiskGB
if existing.GPUEnabled {
currentGPU += existing.GPUCount
if instanceModeUsesDedicatedResources(modeForExistingInstance(&existing)) {
currentCPU += existing.CPUCores
currentMemory += existing.MemoryGB
currentStorage += existing.DiskGB
if existing.GPUEnabled {
currentGPU += existing.GPUCount
}
}
existingNames[strings.TrimSpace(strings.ToLower(existing.Name))] = struct{}{}
}
@@ -102,11 +106,13 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn
return fmt.Errorf("instance name already exists")
}
requestNames[normalizedName] = struct{}{}
requestedCPU += req.CPUCores
requestedMemory += req.MemoryGB
requestedStorage += req.DiskGB
if req.GPUEnabled {
requestedGPU += req.GPUCount
if instanceModeUsesDedicatedResources(resolveCreateInstanceMode(req)) {
requestedCPU += req.CPUCores
requestedMemory += req.MemoryGB
requestedStorage += req.DiskGB
if req.GPUEnabled {
requestedGPU += req.GPUCount
}
}
}
@@ -130,8 +136,10 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn
type CreateInstanceRequest struct {
Name string `json:"name" validate:"required,min=3,max=50"`
Description *string `json:"description,omitempty"`
Type string `json:"type" validate:"required,oneof=openclaw ubuntu debian centos custom webtop hermes"`
RuntimeType string `json:"runtime_type" validate:"omitempty,oneof=desktop shell"`
Type string `json:"type" validate:"required,oneof=openclaw hermes"`
Mode string `json:"mode" validate:"omitempty,oneof=lite pro"`
InstanceMode string `json:"instance_mode" validate:"omitempty,oneof=lite pro"`
RuntimeType string `json:"runtime_type" validate:"omitempty,oneof=gateway desktop shell"`
CPUCores float64 `json:"cpu_cores" validate:"required,min=0.1,max=32"`
MemoryGB int `json:"memory_gb" validate:"required,min=1,max=128"`
DiskGB int `json:"disk_gb" validate:"required,min=10,max=1000"`
@@ -159,6 +167,14 @@ type TeamInstanceConfig struct {
SharedUmask string
}
type instanceModeLimitConfig struct {
Capacity *int
MaxCPU *float64
MaxMemoryGB *int
MaxStorageGB *int
MaxGPUCount *int
}
// UpdateInstanceRequest holds data for updating an instance
type UpdateInstanceRequest struct {
Name *string `json:"name,omitempty" validate:"omitempty,min=3,max=50"`
@@ -167,14 +183,17 @@ type UpdateInstanceRequest struct {
// InstanceStatus holds the status of an instance
type InstanceStatus struct {
InstanceID int `json:"instance_id"`
Status string `json:"status"`
PodName *string `json:"pod_name,omitempty"`
PodNamespace *string `json:"pod_namespace,omitempty"`
PodIP *string `json:"pod_ip,omitempty"`
PodStatus string `json:"pod_status,omitempty"`
CreatedAt time.Time `json:"created_at"`
StartedAt *time.Time `json:"started_at,omitempty"`
InstanceID int `json:"instance_id"`
Status string `json:"status"`
Availability string `json:"availability,omitempty"`
AgentType string `json:"agent_type,omitempty"`
WorkspaceUsageBytes int64 `json:"workspace_usage_bytes,omitempty"`
PodName *string `json:"pod_name,omitempty"`
PodNamespace *string `json:"pod_namespace,omitempty"`
PodIP *string `json:"pod_ip,omitempty"`
PodStatus string `json:"pod_status,omitempty"`
CreatedAt time.Time `json:"created_at"`
StartedAt *time.Time `json:"started_at,omitempty"`
}
// instanceService implements InstanceService
@@ -184,7 +203,12 @@ type instanceService struct {
llmModelRepo repository.LLMModelRepository
openClawConfigService OpenClawConfigService
allowPrivilegedPods bool
runtimePodRepo repository.RuntimePodRepository
bindingRepo repository.InstanceRuntimeBindingRepository
agentClient RuntimeAgentClient
workspaceRoot string
podService *k8s.PodService
deploymentService *k8s.InstanceDeploymentService
pvcService *k8s.PVCService
serviceService *k8s.ServiceService
networkPolicyService *k8s.NetworkPolicyService
@@ -203,6 +227,17 @@ func WithPrivilegedInstancePods(allowed bool) InstanceServiceOption {
}
}
func WithV2RuntimeLifecycle(runtimePodRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, agentClient RuntimeAgentClient, workspaceRoot string) InstanceServiceOption {
return func(s *instanceService) {
s.runtimePodRepo = runtimePodRepo
s.bindingRepo = bindingRepo
s.agentClient = agentClient
if strings.TrimSpace(workspaceRoot) != "" {
s.workspaceRoot = strings.TrimSpace(workspaceRoot)
}
}
}
// NewInstanceService creates a new instance service
func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo repository.QuotaRepository, llmModelRepo repository.LLMModelRepository, openClawConfigService OpenClawConfigService, options ...InstanceServiceOption) InstanceService {
service := &instanceService{
@@ -210,7 +245,9 @@ func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo re
quotaRepo: quotaRepo,
llmModelRepo: llmModelRepo,
openClawConfigService: openClawConfigService,
workspaceRoot: "/workspaces",
podService: k8s.NewPodService(),
deploymentService: k8s.NewInstanceDeploymentService(),
pvcService: k8s.NewPVCService(),
serviceService: k8s.NewServiceService(),
networkPolicyService: k8s.NewNetworkPolicyService(),
@@ -227,6 +264,7 @@ func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo re
func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models.Instance, error) {
ctx := context.Background()
req.Name = strings.TrimSpace(req.Name)
req.Type = strings.ToLower(strings.TrimSpace(req.Type))
environmentOverrides, err := normalizeEnvironmentOverrides(req.EnvironmentOverrides)
if err != nil {
return nil, err
@@ -245,6 +283,11 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
if quota == nil {
return nil, fmt.Errorf("user quota not found")
}
instanceMode := resolveCreateInstanceMode(req)
modeRuntimeType, _ := RuntimeTypeForInstanceMode(instanceMode)
if !hasExplicitCreateInstanceMode(req) && normalizeInstanceRuntimeType(req.RuntimeType) == RuntimeBackendShell {
modeRuntimeType = RuntimeBackendShell
}
// Check instance count limit
currentCount, err := s.instanceRepo.CountByUserID(userID)
@@ -266,11 +309,13 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
currentStorage := 0
currentGPU := 0
for _, existing := range existingInstances {
currentCPU += existing.CPUCores
currentMemory += existing.MemoryGB
currentStorage += existing.DiskGB
if existing.GPUEnabled {
currentGPU += existing.GPUCount
if instanceModeUsesDedicatedResources(modeForExistingInstance(&existing)) {
currentCPU += existing.CPUCores
currentMemory += existing.MemoryGB
currentStorage += existing.DiskGB
if existing.GPUEnabled {
currentGPU += existing.GPUCount
}
}
}
@@ -282,43 +327,58 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
return nil, fmt.Errorf("instance name already exists")
}
// Check CPU limit
if currentCPU+req.CPUCores > quota.MaxCPUCores {
return nil, fmt.Errorf("CPU cores exceed quota: current %v, requested %v, max %v", currentCPU, req.CPUCores, quota.MaxCPUCores)
}
// Check memory limit
if currentMemory+req.MemoryGB > quota.MaxMemoryGB {
return nil, fmt.Errorf("memory exceed quota: current %dGB, requested %dGB, max %dGB", currentMemory, req.MemoryGB, quota.MaxMemoryGB)
}
// Check storage limit
if currentStorage+req.DiskGB > quota.MaxStorageGB {
return nil, fmt.Errorf("storage exceed quota: current %dGB, requested %dGB, max %dGB", currentStorage, req.DiskGB, quota.MaxStorageGB)
}
// Check GPU limit
requestedGPU := 0
if req.GPUEnabled {
requestedGPU = req.GPUCount
}
if currentGPU+requestedGPU > quota.MaxGPUCount {
return nil, fmt.Errorf("GPU count exceed quota: current %d, requested %d, max %d", currentGPU, requestedGPU, quota.MaxGPUCount)
if instanceModeUsesDedicatedResources(instanceMode) {
// Check CPU limit
if currentCPU+req.CPUCores > quota.MaxCPUCores {
return nil, fmt.Errorf("CPU cores exceed quota: current %v, requested %v, max %v", currentCPU, req.CPUCores, quota.MaxCPUCores)
}
// Check memory limit
if currentMemory+req.MemoryGB > quota.MaxMemoryGB {
return nil, fmt.Errorf("memory exceed quota: current %dGB, requested %dGB, max %dGB", currentMemory, req.MemoryGB, quota.MaxMemoryGB)
}
// Check storage limit
if currentStorage+req.DiskGB > quota.MaxStorageGB {
return nil, fmt.Errorf("storage exceed quota: current %dGB, requested %dGB, max %dGB", currentStorage, req.DiskGB, quota.MaxStorageGB)
}
// Check GPU limit
if currentGPU+requestedGPU > quota.MaxGPUCount {
return nil, fmt.Errorf("GPU count exceed quota: current %d, requested %d, max %d", currentGPU, requestedGPU, quota.MaxGPUCount)
}
}
if err := s.enforceInstanceModeLimits(ctx, instanceMode, req.CPUCores, req.MemoryGB, req.DiskGB, requestedGPU); err != nil {
return nil, err
}
if runtimeType, isV2 := NormalizeV2RuntimeType(req.Type); isV2 && instanceMode == InstanceModeLite {
return s.createV2Instance(ctx, userID, req, runtimeType, environmentOverridesJSON)
}
runtimeConfig := buildRuntimeConfig(req.Type, req.OSType, req.OSVersion, req.ImageRegistry, req.ImageTag)
runtimeType := normalizeInstanceRuntimeType(req.RuntimeType)
if modeRuntimeType != "" {
runtimeType = modeRuntimeType
}
if (req.ImageRegistry == nil || strings.TrimSpace(*req.ImageRegistry) == "") && (req.ImageTag == nil || strings.TrimSpace(*req.ImageTag) == "") {
if selection, ok := runtimeImageOverride(req.Type); ok {
image := selection.Image
req.ImageRegistry = &image
req.ImageTag = nil
runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType)
if modeRuntimeType == "" {
runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType)
}
runtimeConfig = buildRuntimeConfig(req.Type, req.OSType, req.OSVersion, req.ImageRegistry, req.ImageTag)
}
} else if req.ImageRegistry != nil {
if selection, ok := runtimeImageOverrideForImage(req.Type, *req.ImageRegistry); ok {
runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType)
if modeRuntimeType == "" {
runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType)
}
}
}
@@ -334,6 +394,7 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
Description: req.Description,
Type: req.Type,
RuntimeType: runtimeType,
InstanceMode: InstanceModeForRuntimeType(runtimeType),
Status: "creating",
CPUCores: req.CPUCores,
MemoryGB: req.MemoryGB,
@@ -506,18 +567,29 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
SecurityMode: s.securityModeForInstance(instance.Type),
}
pod, err := s.podService.CreatePod(ctx, podConfig)
if err != nil {
// Rollback: delete PVC and instance record
s.pvcService.DeletePVC(ctx, userID, instance.ID)
if bootstrapSnapshot != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
}
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to create pod: %w", err)
}
var workloadNamespace string
var workloadName string
if instanceUsesDesktopRuntime(instance) {
if s.deploymentService == nil {
s.pvcService.DeletePVC(ctx, userID, instance.ID)
if bootstrapSnapshot != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, fmt.Errorf("instance deployment service is not configured"))
}
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("instance deployment service is not configured")
}
deployment, err := s.deploymentService.EnsureDeployment(ctx, podConfig, 1)
if err != nil {
s.pvcService.DeletePVC(ctx, userID, instance.ID)
if bootstrapSnapshot != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
}
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to create deployment: %w", err)
}
workloadNamespace = deployment.Namespace
workloadName = deployment.Name
// Create Service for browser desktop access.
serviceConfig := k8s.ServiceConfig{
InstanceID: instance.ID,
@@ -529,8 +601,8 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
serviceInfo, err := s.serviceService.CreateService(ctx, serviceConfig)
if err != nil {
// Rollback: delete pod, PVC and instance record
s.podService.DeletePod(ctx, userID, instance.ID)
// Rollback: delete Deployment, PVC and instance record.
_ = s.deploymentService.DeleteDeployment(ctx, userID, instance.ID)
s.pvcService.DeletePVC(ctx, userID, instance.ID)
if bootstrapSnapshot != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
@@ -541,12 +613,25 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
fmt.Printf("Instance %d: Service created successfully (ClusterIP: %s)\n", instance.ID, serviceInfo.ClusterIP)
} else {
pod, err := s.podService.CreatePod(ctx, podConfig)
if err != nil {
// Rollback: delete PVC and instance record.
s.pvcService.DeletePVC(ctx, userID, instance.ID)
if bootstrapSnapshot != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
}
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to create pod: %w", err)
}
workloadNamespace = pod.Namespace
workloadName = pod.Name
fmt.Printf("Instance %d: Shell runtime selected, skipping desktop service creation\n", instance.ID)
}
// Update instance with pod info
podNamespace := pod.Namespace
podName := pod.Name
// Update instance with initial workload info. For Pro instances this is the
// stable Deployment name; sync later records the active Pod name/IP.
podNamespace := workloadNamespace
podName := workloadName
instance.PodNamespace = &podNamespace
instance.PodName = &podName
instance.Status = "creating"
@@ -576,6 +661,54 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
return instance, nil
}
func (s *instanceService) createV2Instance(ctx context.Context, userID int, req CreateInstanceRequest, runtimeType string, environmentOverridesJSON *string) (*models.Instance, error) {
now := time.Now()
workspaceRoot := s.runtimeWorkspaceRoot()
instance := &models.Instance{
UserID: userID,
Name: strings.TrimSpace(req.Name),
Description: trimOptionalString(req.Description),
Type: runtimeType,
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
Status: "creating",
CPUCores: req.CPUCores,
MemoryGB: req.MemoryGB,
DiskGB: req.DiskGB,
GPUEnabled: req.GPUEnabled,
GPUCount: req.GPUCount,
OSType: req.OSType,
OSVersion: req.OSVersion,
ImageRegistry: req.ImageRegistry,
ImageTag: req.ImageTag,
EnvironmentOverridesJSON: environmentOverridesJSON,
StorageClass: strings.TrimSpace(req.StorageClass),
MountPath: workspaceRoot,
RuntimeGeneration: 1,
CreatedAt: now,
UpdatedAt: now,
StartedAt: &now,
}
if err := s.instanceRepo.Create(instance); err != nil {
return nil, fmt.Errorf("failed to create instance record: %w", err)
}
workspacePath, err := ensureRuntimeWorkspaceDirectories(workspaceRoot, runtimeType, userID, instance.ID)
if err != nil {
_ = s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to create instance workspace: %w", err)
}
if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
_ = s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to persist instance workspace path: %w", err)
}
instance.WorkspacePath = &workspacePath
GetHub().BroadcastInstanceStatus(userID, instance)
return instance, nil
}
// GetByID gets an instance by ID
func (s *instanceService) GetByID(id int) (*models.Instance, error) {
return s.instanceRepo.GetByID(id)
@@ -626,6 +759,13 @@ func (s *instanceService) Start(instanceID int) error {
if instance.Status == "running" {
return fmt.Errorf("instance is already running")
}
if err := s.enforceInstanceModeLimits(ctx, modeForExistingInstance(instance), instance.CPUCores, instance.MemoryGB, instance.DiskGB, instance.GPUCount); err != nil {
return err
}
if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok {
return s.startV2Instance(ctx, instance, runtimeType)
}
if _, err := s.ensureGatewayToken(instance); err != nil {
return fmt.Errorf("failed to provision instance gateway token: %w", err)
@@ -686,12 +826,19 @@ func (s *instanceService) Start(instanceID int) error {
SecurityMode: s.securityModeForInstance(instance.Type),
}
pod, err := s.podService.CreatePod(ctx, podConfig)
if err != nil {
return fmt.Errorf("failed to create pod: %w", err)
}
var workloadNamespace string
var workloadName string
if instanceUsesDesktopRuntime(instance) {
if s.deploymentService == nil {
return fmt.Errorf("instance deployment service is not configured")
}
deployment, err := s.deploymentService.EnsureDeployment(ctx, podConfig, 1)
if err != nil {
return fmt.Errorf("failed to ensure deployment: %w", err)
}
workloadNamespace = deployment.Namespace
workloadName = deployment.Name
// Ensure Service exists (create if not exists)
serviceExists, _ := s.serviceService.ServiceExists(ctx, instance.UserID, instance.ID)
if !serviceExists {
@@ -708,12 +855,19 @@ func (s *instanceService) Start(instanceID int) error {
// Don't fail if service creation fails, pod is already running
}
}
} else {
pod, err := s.podService.CreatePod(ctx, podConfig)
if err != nil {
return fmt.Errorf("failed to create pod: %w", err)
}
workloadNamespace = pod.Namespace
workloadName = pod.Name
}
// Update instance status
now := time.Now()
podNamespace := pod.Namespace
podName := pod.Name
podNamespace := workloadNamespace
podName := workloadName
instance.PodNamespace = &podNamespace
instance.PodName = &podName
instance.Status = "creating"
@@ -792,6 +946,25 @@ func (s *instanceService) buildGatewayEnv(instance *models.Instance) (map[string
}, nil
}
func (s *instanceService) BuildGatewayEnv(instance *models.Instance) (map[string]string, error) {
if instance == nil || !supportsManagedRuntimeIntegration(instance.Type) {
return s.buildGatewayEnv(instance)
}
if instance.AccessToken == nil || strings.TrimSpace(*instance.AccessToken) == "" {
if s == nil || s.instanceRepo == nil {
return nil, fmt.Errorf("instance repository is not configured")
}
if _, err := s.ensureGatewayToken(instance); err != nil {
return nil, err
}
}
gatewayEnv, err := s.buildGatewayEnv(instance)
if err != nil {
return nil, err
}
return buildInstanceGatewayEnv(instance, gatewayEnv)
}
func (s *instanceService) ensureAgentBootstrapToken(instance *models.Instance) (string, error) {
if instance.AgentBootstrapToken != nil && strings.TrimSpace(*instance.AgentBootstrapToken) != "" {
return strings.TrimSpace(*instance.AgentBootstrapToken), nil
@@ -964,6 +1137,130 @@ func mergeEnvMaps(base map[string]string, overlay map[string]string) map[string]
return merged
}
func (s *instanceService) startV2Instance(ctx context.Context, instance *models.Instance, runtimeType string) error {
if err := s.ensureV2Workspace(ctx, instance, runtimeType); err != nil {
return err
}
nextGeneration := instance.RuntimeGeneration + 1
if nextGeneration <= 0 {
nextGeneration = 1
}
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "creating", nextGeneration, nil); err != nil {
return fmt.Errorf("failed to mark v2 instance creating: %w", err)
}
instance.Status = "creating"
instance.RuntimeGeneration = nextGeneration
instance.RuntimeErrorMessage = nil
now := time.Now()
instance.StartedAt = &now
instance.UpdatedAt = now
GetHub().BroadcastInstanceStatus(instance.UserID, instance)
return nil
}
func (s *instanceService) stopV2Instance(ctx context.Context, instance *models.Instance) error {
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "stopped", instance.RuntimeGeneration, nil); err != nil {
return fmt.Errorf("failed to mark v2 instance stopped: %w", err)
}
now := time.Now()
instance.Status = "stopped"
instance.StoppedAt = &now
instance.PodName = nil
instance.PodNamespace = nil
instance.PodIP = nil
instance.UpdatedAt = now
GetHub().BroadcastInstanceStatus(instance.UserID, instance)
return s.cleanupV2GatewayBinding(ctx, instance)
}
func (s *instanceService) deleteV2Instance(ctx context.Context, instance *models.Instance) error {
if instance.Status != "deleting" {
now := time.Now()
instance.Status = "deleting"
instance.UpdatedAt = now
if err := s.instanceRepo.Update(instance); err != nil {
return fmt.Errorf("failed to mark v2 instance as deleting: %w", err)
}
GetHub().BroadcastInstanceStatus(instance.UserID, instance)
}
cleanupErr := s.cleanupV2GatewayBinding(ctx, instance)
if cleanupErr != nil {
return cleanupErr
}
if err := s.instanceRepo.Delete(instance.ID); err != nil {
return fmt.Errorf("failed to delete v2 instance record: %w", err)
}
return nil
}
func (s *instanceService) cleanupV2GatewayBinding(ctx context.Context, instance *models.Instance) error {
if s.bindingRepo == nil {
return nil
}
binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
if err != nil {
return fmt.Errorf("failed to get v2 runtime binding: %w", err)
}
if binding == nil {
return nil
}
if s.runtimePodRepo != nil {
pod, podErr := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
if podErr != nil {
return fmt.Errorf("failed to get runtime pod %d for v2 cleanup: %w", binding.RuntimePodID, podErr)
} else if pod == nil {
return fmt.Errorf("runtime pod %d is not available for v2 cleanup", binding.RuntimePodID)
} else if pod != nil && pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" && s.agentClient != nil && binding.GatewayID != "" {
if err := s.agentClient.DeleteGateway(ctx, strings.TrimSpace(*pod.AgentEndpoint), binding.GatewayID); err != nil {
return fmt.Errorf("failed to delete v2 gateway: %w", err)
}
}
}
if err := s.bindingRepo.DeleteByInstanceIDAndReleaseSlot(ctx, instance.ID, binding.RuntimePodID); err != nil {
return fmt.Errorf("failed to delete v2 runtime binding and release slot: %w", err)
}
return nil
}
func (s *instanceService) ensureV2Workspace(ctx context.Context, instance *models.Instance, runtimeType string) error {
if instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != "" {
return nil
}
workspacePath, err := ensureRuntimeWorkspaceDirectories(s.runtimeWorkspaceRoot(), runtimeType, instance.UserID, instance.ID)
if err != nil {
return fmt.Errorf("failed to create instance workspace: %w", err)
}
if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
return fmt.Errorf("failed to persist instance workspace path: %w", err)
}
instance.WorkspacePath = &workspacePath
return nil
}
func ensureRuntimeWorkspaceDirectories(root, runtimeType string, userID, instanceID int) (string, error) {
workspacePath := RuntimeWorkspacePathWithRoot(root, runtimeType, userID, instanceID)
if err := os.MkdirAll(workspacePath, 0750); err != nil {
return "", err
}
// Allow the isolated gateway UID to traverse to its own workspace without
// granting read/list access to sibling user or instance directories.
userRoot := path.Dir(workspacePath)
runtimeRoot := path.Dir(userRoot)
for _, dir := range []string{runtimeRoot, userRoot} {
if err := os.Chmod(dir, 0711); err != nil {
return "", err
}
}
if err := os.Chmod(workspacePath, 0750); err != nil {
return "", err
}
return workspacePath, nil
}
// Stop stops an instance
func (s *instanceService) Stop(instanceID int) error {
ctx := context.Background()
@@ -977,13 +1274,29 @@ func (s *instanceService) Stop(instanceID int) error {
return fmt.Errorf("instance not found")
}
if _, ok := v2RuntimeTypeForInstance(instance); ok {
return s.stopV2Instance(ctx, instance)
}
if instance.Status != "running" {
return fmt.Errorf("instance is not running")
}
// Delete pod
if err := s.podService.DeletePod(ctx, instance.UserID, instance.ID); err != nil {
return fmt.Errorf("failed to delete pod: %w", err)
if instanceUsesDesktopRuntime(instance) {
if s.deploymentService == nil {
return fmt.Errorf("instance deployment service is not configured")
}
if err := s.deploymentService.ScaleDeployment(ctx, instance.UserID, instance.ID, 0); err != nil {
fmt.Printf("Warning: failed to stop deployment for instance %d, falling back to pod delete: %v\n", instance.ID, err)
if podErr := s.podService.DeletePod(ctx, instance.UserID, instance.ID); podErr != nil {
return fmt.Errorf("failed to stop deployment: %w", err)
}
}
} else {
// Delete shell pod
if err := s.podService.DeletePod(ctx, instance.UserID, instance.ID); err != nil {
return fmt.Errorf("failed to delete pod: %w", err)
}
}
// Update instance status
@@ -1029,6 +1342,10 @@ func (s *instanceService) Delete(instanceID int) error {
return fmt.Errorf("instance not found")
}
if _, ok := v2RuntimeTypeForInstance(instance); ok {
return s.deleteV2Instance(context.Background(), instance)
}
if instance.Status != "deleting" {
now := time.Now()
instance.Status = "deleting"
@@ -1300,6 +1617,18 @@ func (s *instanceService) GetInstanceStatus(instanceID int) (*InstanceStatus, er
return nil, fmt.Errorf("instance not found")
}
if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok {
return &InstanceStatus{
InstanceID: instance.ID,
Status: instance.Status,
Availability: s.v2InstanceAvailability(ctx, instance),
AgentType: runtimeType,
WorkspaceUsageBytes: instance.WorkspaceUsageBytes,
CreatedAt: instance.CreatedAt,
StartedAt: instance.StartedAt,
}, nil
}
status := &InstanceStatus{
InstanceID: instance.ID,
Status: instance.Status,
@@ -1334,6 +1663,13 @@ func (s *instanceService) ForceSyncInstance(instanceID int) error {
return fmt.Errorf("instance not found")
}
if _, ok := v2RuntimeTypeForInstance(instance); ok {
return nil
}
if instanceUsesDesktopRuntime(instance) {
return s.forceSyncDeploymentInstance(ctx, instance)
}
fmt.Printf("Force syncing instance %d (current status: %s, user: %d)\n", instanceID, instance.Status, instance.UserID)
// First try direct lookup by instance ID
@@ -1452,6 +1788,60 @@ func (s *instanceService) ForceSyncInstance(instanceID int) error {
return nil
}
func (s *instanceService) forceSyncDeploymentInstance(ctx context.Context, instance *models.Instance) error {
if s.deploymentService == nil {
return fmt.Errorf("instance deployment service is not configured")
}
deployment, err := s.deploymentService.GetDeployment(ctx, instance.UserID, instance.ID)
if err != nil {
if instance.Status == "running" || instance.Status == "creating" {
nextStatus := "stopped"
if instance.Status == "creating" {
nextStatus = "error"
}
instance.Status = nextStatus
instance.PodName = nil
instance.PodNamespace = nil
instance.PodIP = nil
instance.UpdatedAt = time.Now()
if err := s.instanceRepo.Update(instance); err != nil {
return fmt.Errorf("failed to update instance status: %w", err)
}
GetHub().BroadcastInstanceStatus(instance.UserID, instance)
}
return nil
}
needsUpdate := false
desiredStatus := mapDeploymentToInstanceStatus(deployment)
if instance.Status != desiredStatus {
instance.Status = desiredStatus
needsUpdate = true
}
if pod, podErr := s.deploymentService.GetActivePod(ctx, instance.UserID, instance.ID); podErr == nil && pod != nil {
if pod.Status.PodIP != "" && (instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP) {
instance.PodIP = &pod.Status.PodIP
needsUpdate = true
}
if instance.PodName == nil || *instance.PodName != pod.Name {
instance.PodName = &pod.Name
needsUpdate = true
}
if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace {
instance.PodNamespace = &pod.Namespace
needsUpdate = true
}
}
if needsUpdate {
instance.UpdatedAt = time.Now()
if err := s.instanceRepo.Update(instance); err != nil {
return fmt.Errorf("failed to update instance: %w", err)
}
GetHub().BroadcastInstanceStatus(instance.UserID, instance)
}
return nil
}
func additionalServicePorts(primaryPort int32) []int32 {
if primaryPort == 3000 || primaryPort == 8082 {
return []int32{3000, 8082}
@@ -1462,10 +1852,12 @@ func additionalServicePorts(primaryPort int32) []int32 {
func normalizeInstanceRuntimeType(runtimeType string) string {
switch strings.ToLower(strings.TrimSpace(runtimeType)) {
case RuntimeBackendGateway:
return RuntimeBackendGateway
case "shell":
return "shell"
return RuntimeBackendShell
default:
return "desktop"
return RuntimeBackendDesktop
}
}
@@ -1473,5 +1865,182 @@ func instanceUsesDesktopRuntime(instance *models.Instance) bool {
if instance == nil {
return true
}
return normalizeInstanceRuntimeType(instance.RuntimeType) == "desktop"
return normalizeInstanceRuntimeType(instance.RuntimeType) == RuntimeBackendDesktop
}
func resolveCreateInstanceMode(req CreateInstanceRequest) string {
if mode, ok := NormalizeInstanceMode(req.Mode); ok {
return mode
}
if mode, ok := NormalizeInstanceMode(req.InstanceMode); ok {
return mode
}
if strings.TrimSpace(req.RuntimeType) == "" {
return InstanceModeLite
}
return InstanceModeForRuntimeType(normalizeInstanceRuntimeType(req.RuntimeType))
}
func hasExplicitCreateInstanceMode(req CreateInstanceRequest) bool {
if _, ok := NormalizeInstanceMode(req.Mode); ok {
return true
}
if _, ok := NormalizeInstanceMode(req.InstanceMode); ok {
return true
}
return false
}
func modeForExistingInstance(instance *models.Instance) string {
if instance == nil {
return InstanceModeLite
}
if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok {
return mode
}
return InstanceModeForRuntimeType(normalizeInstanceRuntimeType(instance.RuntimeType))
}
func instanceModeUsesDedicatedResources(mode string) bool {
normalized, ok := NormalizeInstanceMode(mode)
return ok && normalized == InstanceModePro
}
func (s *instanceService) enforceInstanceModeLimits(ctx context.Context, mode string, cpuCores float64, memoryGB, storageGB, gpuCount int) error {
normalizedMode, ok := NormalizeInstanceMode(mode)
if !ok {
return fmt.Errorf("unsupported instance mode %q", mode)
}
limits := loadInstanceModeLimitConfig(normalizedMode)
if limits.Capacity != nil {
if *limits.Capacity <= 0 {
return fmt.Errorf("%s instance mode is disabled", normalizedMode)
}
if s == nil || s.instanceRepo == nil {
return fmt.Errorf("instance repository is not configured")
}
count, err := s.instanceRepo.CountActiveByMode(ctx, normalizedMode)
if err != nil {
return err
}
if count >= *limits.Capacity {
return fmt.Errorf("%s instance capacity reached: %d/%d", normalizedMode, count, *limits.Capacity)
}
}
if !instanceModeUsesDedicatedResources(normalizedMode) {
return nil
}
if limits.MaxCPU != nil && cpuCores > *limits.MaxCPU {
return fmt.Errorf("%s CPU cores exceed mode limit: requested %g, max %g", normalizedMode, cpuCores, *limits.MaxCPU)
}
if limits.MaxMemoryGB != nil && memoryGB > *limits.MaxMemoryGB {
return fmt.Errorf("%s memory exceeds mode limit: requested %dGB, max %dGB", normalizedMode, memoryGB, *limits.MaxMemoryGB)
}
if limits.MaxStorageGB != nil && storageGB > *limits.MaxStorageGB {
return fmt.Errorf("%s storage exceeds mode limit: requested %dGB, max %dGB", normalizedMode, storageGB, *limits.MaxStorageGB)
}
if limits.MaxGPUCount != nil && gpuCount > *limits.MaxGPUCount {
return fmt.Errorf("%s GPU count exceeds mode limit: requested %d, max %d", normalizedMode, gpuCount, *limits.MaxGPUCount)
}
return nil
}
func loadInstanceModeLimitConfig(mode string) instanceModeLimitConfig {
prefix := "CLAWMANAGER_" + strings.ToUpper(mode) + "_"
return instanceModeLimitConfig{
Capacity: optionalIntEnv(prefix + "CAPACITY"),
MaxCPU: optionalFloatEnv(prefix + "MAX_CPU_CORES"),
MaxMemoryGB: optionalIntEnv(prefix + "MAX_MEMORY_GB"),
MaxStorageGB: optionalIntEnv(prefix + "MAX_STORAGE_GB"),
MaxGPUCount: optionalIntEnv(prefix + "MAX_GPU_COUNT"),
}
}
func optionalIntEnv(key string) *int {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return nil
}
parsed, err := strconv.Atoi(value)
if err != nil {
return nil
}
return &parsed
}
func optionalFloatEnv(key string) *float64 {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return nil
}
parsed, err := strconv.ParseFloat(value, 64)
if err != nil {
return nil
}
return &parsed
}
func (s *instanceService) runtimeWorkspaceRoot() string {
if s != nil && strings.TrimSpace(s.workspaceRoot) != "" {
return strings.TrimSpace(s.workspaceRoot)
}
return "/workspaces"
}
func v2RuntimeTypeForInstance(instance *models.Instance) (string, bool) {
if instance == nil {
return "", false
}
runtimeType, ok := NormalizeV2RuntimeType(instance.Type)
if !ok {
return "", false
}
if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), RuntimeBackendGateway) {
return runtimeType, true
}
if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok && mode == InstanceModeLite {
return runtimeType, true
}
return "", false
}
func availabilityForStatus(status string) string {
switch strings.ToLower(strings.TrimSpace(status)) {
case "running":
return "available"
case "creating":
return "starting"
default:
return "unavailable"
}
}
func (s *instanceService) v2InstanceAvailability(ctx context.Context, instance *models.Instance) string {
base := availabilityForStatus(instance.Status)
if base != "available" {
return base
}
if s == nil || s.bindingRepo == nil || s.runtimePodRepo == nil {
return "unavailable"
}
binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID)
if err != nil || binding == nil || binding.GatewayPort <= 0 {
return "unavailable"
}
pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
if err != nil || pod == nil || pod.PodIP == nil || strings.TrimSpace(*pod.PodIP) == "" {
return "unavailable"
}
return "available"
}
func trimOptionalString(value *string) *string {
if value == nil {
return nil
}
trimmed := strings.TrimSpace(*value)
if trimmed == "" {
return nil
}
return &trimmed
}
backend/internal/services/instance_service_test.go
+87
View File
@@ -80,6 +80,79 @@ func TestBuildGatewayEnvInjectsGatewayModelCatalog(t *testing.T) {
}
}
func TestBuildGatewayEnvEnsuresMissingGatewayToken(t *testing.T) {
t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm")
instanceRepo := &stubGatewayEnvInstanceRepository{fakeRuntimeInstanceRepo: newFakeRuntimeInstanceRepo()}
service := &instanceService{
instanceRepo: instanceRepo,
llmModelRepo: &stubLLMModelRepository{
active: []models.LLMModel{{DisplayName: "auto"}},
},
}
instance := &models.Instance{
ID: 68,
UserID: 1,
Type: "openclaw",
}
env, err := service.BuildGatewayEnv(instance)
if err != nil {
t.Fatalf("BuildGatewayEnv returned error: %v", err)
}
if instance.AccessToken == nil || *instance.AccessToken == "" {
t.Fatal("BuildGatewayEnv did not provision instance access token")
}
if env["CLAWMANAGER_LLM_API_KEY"] != *instance.AccessToken {
t.Fatalf("CLAWMANAGER_LLM_API_KEY = %q, want provisioned token %q", env["CLAWMANAGER_LLM_API_KEY"], *instance.AccessToken)
}
if got := instanceRepo.updated[68]; got == nil || got.AccessToken == nil || *got.AccessToken != *instance.AccessToken {
t.Fatalf("repository update did not persist provisioned token: %#v", instanceRepo.updated[68])
}
}
func TestBuildGatewayEnvMergesEnvironmentOverrides(t *testing.T) {
t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm")
token := "igt_test_token"
raw, err := marshalEnvironmentOverrides(map[string]string{
"CLAWMANAGER_TEAM_ENABLED": "true",
"CLAWMANAGER_TEAM_MEMBER_ID": "lite-worker",
"CUSTOM_GATEWAY_ENV": "enabled",
})
if err != nil {
t.Fatalf("marshalEnvironmentOverrides returned error: %v", err)
}
service := &instanceService{
llmModelRepo: &stubLLMModelRepository{
active: []models.LLMModel{{DisplayName: "auto"}},
},
}
env, err := service.BuildGatewayEnv(&models.Instance{
ID: 88,
Type: "openclaw",
RuntimeType: RuntimeBackendGateway,
AccessToken: &token,
EnvironmentOverridesJSON: raw,
})
if err != nil {
t.Fatalf("BuildGatewayEnv returned error: %v", err)
}
if env["CLAWMANAGER_TEAM_ENABLED"] != "true" || env["CLAWMANAGER_TEAM_MEMBER_ID"] != "lite-worker" {
t.Fatalf("expected Team environment overrides to be merged into gateway env, got %#v", env)
}
if env["CUSTOM_GATEWAY_ENV"] != "enabled" {
t.Fatalf("expected custom gateway environment override to be merged, got %#v", env)
}
if env["CLAWMANAGER_LLM_API_KEY"] != token {
t.Fatalf("expected gateway token env to remain available")
}
if env["CLAWMANAGER_RUNTIME_TYPE"] != RuntimeBackendGateway {
t.Fatalf("expected runtime type marker %q, got %q", RuntimeBackendGateway, env["CLAWMANAGER_RUNTIME_TYPE"])
}
}
func TestBuildGatewayEnvSkipsUnmanagedRuntime(t *testing.T) {
token := "igt_test_token"
service := &instanceService{}
@@ -96,6 +169,20 @@ func TestBuildGatewayEnvSkipsUnmanagedRuntime(t *testing.T) {
}
}
type stubGatewayEnvInstanceRepository struct {
*fakeRuntimeInstanceRepo
updated map[int]*models.Instance
}
func (r *stubGatewayEnvInstanceRepository) Update(instance *models.Instance) error {
if r.updated == nil {
r.updated = map[int]*models.Instance{}
}
copy := *instance
r.updated[instance.ID] = &copy
return nil
}
func TestBuildAgentEnvInjectsHermesAgentConfig(t *testing.T) {
t.Setenv("CLAWMANAGER_AGENT_CONTROL_BASE_URL", "http://agent-control.example")
backend/internal/services/instance_service_v2_test.go
+751
View File
@@ -0,0 +1,751 @@
package services
import (
"context"
"errors"
"os"
"path"
"strings"
"testing"
"clawreef/internal/models"
)
func TestInstanceServiceCreateV2CreatesWorkspaceOnly(t *testing.T) {
workspaceRoot := strings.ReplaceAll(t.TempDir(), "\\", "/")
instanceRepo := newV2LifecycleInstanceRepo()
service := &instanceService{
instanceRepo: instanceRepo,
quotaRepo: v2LifecycleQuotaRepo{},
workspaceRoot: workspaceRoot,
}
instance, err := service.Create(45, CreateInstanceRequest{
Name: "OpenClaw Dev",
Type: " OpenClaw ",
CPUCores: 2,
MemoryGB: 4,
DiskGB: 20,
OSType: "openclaw",
OSVersion: "latest",
})
if err != nil {
t.Fatalf("Create returned error: %v", err)
}
expectedWorkspacePath := RuntimeWorkspacePathWithRoot(workspaceRoot, "openclaw", 45, instance.ID)
if instance.Type != "openclaw" {
t.Fatalf("instance type = %q, want openclaw", instance.Type)
}
if instance.RuntimeType != "gateway" {
t.Fatalf("runtime type = %q, want gateway", instance.RuntimeType)
}
if instance.InstanceMode != InstanceModeLite {
t.Fatalf("instance mode = %q, want lite", instance.InstanceMode)
}
if instance.Status != "creating" {
t.Fatalf("status = %q, want creating", instance.Status)
}
if instance.RuntimeGeneration != 1 {
t.Fatalf("runtime generation = %d, want 1", instance.RuntimeGeneration)
}
if instance.WorkspacePath == nil || *instance.WorkspacePath != expectedWorkspacePath {
t.Fatalf("workspace path = %v, want %q", instance.WorkspacePath, expectedWorkspacePath)
}
if _, err := os.Stat(expectedWorkspacePath); err != nil {
t.Fatalf("expected workspace directory to exist: %v", err)
}
if os.PathSeparator == '/' {
assertDirMode(t, path.Dir(path.Dir(expectedWorkspacePath)), 0711)
assertDirMode(t, path.Dir(expectedWorkspacePath), 0711)
assertDirMode(t, expectedWorkspacePath, 0750)
}
if instance.PodName != nil || instance.PodNamespace != nil || instance.PodIP != nil {
t.Fatalf("V2 create must not set per-instance pod fields: %#v", instance)
}
if len(instanceRepo.created) != 1 {
t.Fatalf("created instance records = %d, want 1", len(instanceRepo.created))
}
}
func TestInstanceServiceCreateLiteSkipsPerInstanceResourceQuota(t *testing.T) {
workspaceRoot := strings.ReplaceAll(t.TempDir(), "\\", "/")
instanceRepo := newV2LifecycleInstanceRepo()
service := &instanceService{
instanceRepo: instanceRepo,
quotaRepo: fixedV2QuotaRepo{cpu: 1, memory: 1, storage: 10, gpu: 0},
workspaceRoot: workspaceRoot,
}
instance, err := service.Create(45, CreateInstanceRequest{
Name: "Lite Gateway",
Type: "openclaw",
Mode: InstanceModeLite,
RuntimeType: RuntimeBackendGateway,
CPUCores: 8,
MemoryGB: 32,
DiskGB: 200,
GPUEnabled: true,
GPUCount: 1,
OSType: "openclaw",
OSVersion: "latest",
StorageClass: "manual",
})
if err != nil {
t.Fatalf("Create lite returned error: %v", err)
}
if instance.InstanceMode != InstanceModeLite || instance.RuntimeType != RuntimeBackendGateway {
t.Fatalf("created runtime = mode %q type %q, want lite gateway", instance.InstanceMode, instance.RuntimeType)
}
}
func TestInstanceServiceCreateProEnforcesPerInstanceResourceQuota(t *testing.T) {
instanceRepo := newV2LifecycleInstanceRepo()
service := &instanceService{
instanceRepo: instanceRepo,
quotaRepo: fixedV2QuotaRepo{cpu: 1, memory: 1, storage: 10, gpu: 0},
pvcService: nil,
deploymentService: nil,
serviceService: nil,
networkPolicyService: nil,
openClawConfigService: nil,
}
_, err := service.Create(45, CreateInstanceRequest{
Name: "Pro Desktop",
Type: "openclaw",
Mode: InstanceModePro,
RuntimeType: RuntimeBackendDesktop,
CPUCores: 8,
MemoryGB: 32,
DiskGB: 200,
OSType: "openclaw",
OSVersion: "latest",
})
if err == nil || !strings.Contains(err.Error(), "CPU cores exceed quota") {
t.Fatalf("Create pro error = %v, want CPU quota failure", err)
}
}
func assertDirMode(t *testing.T, dir string, want os.FileMode) {
t.Helper()
info, err := os.Stat(dir)
if err != nil {
t.Fatalf("stat %s: %v", dir, err)
}
if got := info.Mode().Perm(); got != want {
t.Fatalf("mode %s = %04o, want %04o", dir, got, want)
}
}
func TestInstanceServiceStartV2MarksCreatingWithNextGeneration(t *testing.T) {
workspacePath := "/workspaces/hermes/user-45/instance-77"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[77] = &models.Instance{
ID: 77,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
Status: "stopped",
WorkspacePath: &workspacePath,
RuntimeGeneration: 4,
}
service := &instanceService{instanceRepo: instanceRepo}
if err := service.Start(77); err != nil {
t.Fatalf("Start returned error: %v", err)
}
state := instanceRepo.runtimeStates[77]
if state.status != "creating" || state.generation != 5 {
t.Fatalf("runtime state = %#v, want creating generation 5", state)
}
}
func TestInstanceServiceStopV2DeletesGatewayBindingAndReleasesSlot(t *testing.T) {
workspacePath := "/workspaces/openclaw/user-45/instance-88"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[88] = &models.Instance{
ID: 88,
UserID: 45,
Type: "openclaw",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
RuntimeGeneration: 3,
}
endpoint := "http://agent.local:19090"
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
9: {ID: 9, AgentEndpoint: &endpoint},
},
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[88] = &models.InstanceRuntimeBinding{
InstanceID: 88,
RuntimePodID: 9,
GatewayID: "gw-88",
GatewayPort: 20018,
State: "running",
Generation: 3,
}
agent := &fakeRuntimeAgentClient{}
service := &instanceService{
instanceRepo: instanceRepo,
runtimePodRepo: podRepo,
bindingRepo: bindingRepo,
agentClient: agent,
workspaceRoot: "/workspaces",
}
if err := service.Stop(88); err != nil {
t.Fatalf("Stop returned error: %v", err)
}
if len(agent.deleteRequests) != 1 || agent.deleteRequests[0].endpoint != endpoint || agent.deleteRequests[0].gatewayID != "gw-88" {
t.Fatalf("delete gateway requests = %#v", agent.deleteRequests)
}
if bindingRepo.deleteAndReleaseCalls[88] != 1 {
t.Fatalf("binding delete and release calls = %d, want 1", bindingRepo.deleteAndReleaseCalls[88])
}
state := instanceRepo.runtimeStates[88]
if state.status != "stopped" || state.generation != 3 {
t.Fatalf("runtime state = %#v, want stopped generation 3", state)
}
}
func TestInstanceServiceStopV2KeepsBindingWhenAgentDeleteFails(t *testing.T) {
workspacePath := "/workspaces/openclaw/user-45/instance-188"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[188] = &models.Instance{
ID: 188,
UserID: 45,
Type: "openclaw",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
RuntimeGeneration: 6,
}
endpoint := "http://agent.local:19090"
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
19: {ID: 19, AgentEndpoint: &endpoint},
},
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[188] = &models.InstanceRuntimeBinding{
InstanceID: 188,
RuntimePodID: 19,
GatewayID: "gw-188",
GatewayPort: 20018,
State: "running",
Generation: 6,
}
agent := &fakeRuntimeAgentClient{deleteErr: errors.New("agent delete failed")}
service := &instanceService{
instanceRepo: instanceRepo,
runtimePodRepo: podRepo,
bindingRepo: bindingRepo,
agentClient: agent,
}
err := service.Stop(188)
if err == nil || !strings.Contains(err.Error(), "failed to delete v2 gateway") {
t.Fatalf("Stop error = %v, want gateway delete failure", err)
}
if bindingRepo.bindings[188] == nil {
t.Fatal("binding was deleted after gateway delete failed")
}
if bindingRepo.deleteAndReleaseCalls[188] != 0 {
t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[188])
}
state := instanceRepo.runtimeStates[188]
if state.status != "stopped" || state.generation != 6 {
t.Fatalf("runtime state = %#v, want stopped generation 6", state)
}
}
func TestInstanceServiceStopV2KeepsBindingWhenRuntimePodLookupFails(t *testing.T) {
workspacePath := "/workspaces/openclaw/user-45/instance-189"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[189] = &models.Instance{
ID: 189,
UserID: 45,
Type: "openclaw",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
RuntimeGeneration: 6,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[189] = &models.InstanceRuntimeBinding{
InstanceID: 189,
RuntimePodID: 29,
GatewayID: "gw-189",
GatewayPort: 20019,
State: "running",
Generation: 6,
}
service := &instanceService{
instanceRepo: instanceRepo,
runtimePodRepo: &fakeRuntimePodRepo{getErr: errors.New("pod lookup failed")},
bindingRepo: bindingRepo,
agentClient: &fakeRuntimeAgentClient{},
}
err := service.Stop(189)
if err == nil || !strings.Contains(err.Error(), "failed to get runtime pod") {
t.Fatalf("Stop error = %v, want pod lookup failure", err)
}
if bindingRepo.bindings[189] == nil {
t.Fatal("binding was deleted after pod lookup failed")
}
if bindingRepo.deleteAndReleaseCalls[189] != 0 {
t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[189])
}
state := instanceRepo.runtimeStates[189]
if state.status != "stopped" || state.generation != 6 {
t.Fatalf("runtime state = %#v, want stopped generation 6", state)
}
}
func TestInstanceServiceRestartV2RecreatesGatewayViaScheduler(t *testing.T) {
workspacePath := "/workspaces/openclaw/user-45/instance-89"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[89] = &models.Instance{
ID: 89,
UserID: 45,
Type: "openclaw",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
RuntimeGeneration: 8,
}
endpoint := "http://agent.local:19090"
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
11: {ID: 11, AgentEndpoint: &endpoint},
},
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[89] = &models.InstanceRuntimeBinding{
InstanceID: 89,
RuntimePodID: 11,
GatewayID: "gw-89",
GatewayPort: 20019,
State: "running",
Generation: 8,
}
agent := &fakeRuntimeAgentClient{}
service := &instanceService{
instanceRepo: instanceRepo,
runtimePodRepo: podRepo,
bindingRepo: bindingRepo,
agentClient: agent,
}
if err := service.Restart(89); err != nil {
t.Fatalf("Restart returned error: %v", err)
}
if len(agent.deleteRequests) != 1 || agent.deleteRequests[0].gatewayID != "gw-89" {
t.Fatalf("delete gateway requests = %#v", agent.deleteRequests)
}
if bindingRepo.deleteAndReleaseCalls[89] != 1 {
t.Fatalf("binding delete and release calls = %d, want 1", bindingRepo.deleteAndReleaseCalls[89])
}
state := instanceRepo.runtimeStates[89]
if state.status != "creating" || state.generation != 9 {
t.Fatalf("runtime state = %#v, want creating generation 9", state)
}
}
func TestInstanceServiceDeleteV2MissingBindingStillDeletesInstance(t *testing.T) {
workspacePath := "/workspaces/hermes/user-45/instance-90"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[90] = &models.Instance{
ID: 90,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
Status: "stopped",
WorkspacePath: &workspacePath,
RuntimeGeneration: 2,
}
service := &instanceService{
instanceRepo: instanceRepo,
runtimePodRepo: &fakeRuntimePodRepo{},
bindingRepo: newFakeRuntimeBindingRepo(),
agentClient: &fakeRuntimeAgentClient{},
}
if err := service.Delete(90); err != nil {
t.Fatalf("Delete returned error: %v", err)
}
if len(instanceRepo.deleted) != 1 || instanceRepo.deleted[0] != 90 {
t.Fatalf("deleted instances = %#v, want [90]", instanceRepo.deleted)
}
}
func TestInstanceServiceDeleteV2KeepsInstanceAndBindingWhenAgentDeleteFails(t *testing.T) {
workspacePath := "/workspaces/hermes/user-45/instance-190"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[190] = &models.Instance{
ID: 190,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
RuntimeGeneration: 2,
}
endpoint := "http://agent.local:19090"
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
39: {ID: 39, AgentEndpoint: &endpoint},
},
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[190] = &models.InstanceRuntimeBinding{
InstanceID: 190,
RuntimePodID: 39,
GatewayID: "gw-190",
GatewayPort: 20090,
State: "running",
Generation: 2,
}
service := &instanceService{
instanceRepo: instanceRepo,
runtimePodRepo: podRepo,
bindingRepo: bindingRepo,
agentClient: &fakeRuntimeAgentClient{deleteErr: errors.New("agent delete failed")},
}
err := service.Delete(190)
if err == nil || !strings.Contains(err.Error(), "failed to delete v2 gateway") {
t.Fatalf("Delete error = %v, want gateway delete failure", err)
}
if len(instanceRepo.deleted) != 0 {
t.Fatalf("deleted instances = %#v, want none", instanceRepo.deleted)
}
if instanceRepo.byID[190] == nil {
t.Fatal("instance was removed after cleanup failure")
}
if bindingRepo.bindings[190] == nil {
t.Fatal("binding was removed after cleanup failure")
}
if bindingRepo.deleteAndReleaseCalls[190] != 0 {
t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[190])
}
}
func TestInstanceServiceV2StatusRequiresRunningBindingAndPodIP(t *testing.T) {
workspacePath := "/workspaces/openclaw/user-45/instance-92"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[92] = &models.Instance{
ID: 92,
UserID: 45,
Type: "openclaw",
RuntimeType: "gateway",
Status: "running",
WorkspacePath: &workspacePath,
WorkspaceUsageBytes: 2048,
RuntimeGeneration: 1,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[92] = &models.InstanceRuntimeBinding{
InstanceID: 92,
RuntimePodID: 12,
GatewayPort: 20020,
State: "running",
}
service := &instanceService{
instanceRepo: instanceRepo,
runtimePodRepo: &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{12: {ID: 12}}},
bindingRepo: bindingRepo,
}
status, err := service.GetInstanceStatus(92)
if err != nil {
t.Fatalf("GetInstanceStatus returned error: %v", err)
}
if status.Availability != "unavailable" {
t.Fatalf("availability without pod IP = %q, want unavailable", status.Availability)
}
podIP := "10.42.0.92"
service.runtimePodRepo = &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{12: {ID: 12, PodIP: &podIP}}}
status, err = service.GetInstanceStatus(92)
if err != nil {
t.Fatalf("GetInstanceStatus returned error after pod IP: %v", err)
}
if status.Availability != "available" {
t.Fatalf("availability with running binding and pod IP = %q, want available", status.Availability)
}
}
func TestSyncInstanceSkipsV2RuntimeGatewayInstances(t *testing.T) {
workspacePath := "/workspaces/openclaw/user-45/instance-91"
instanceRepo := newV2LifecycleInstanceRepo()
service := &SyncService{instanceRepo: instanceRepo}
instance := &models.Instance{
ID: 91,
UserID: 45,
Type: "openclaw",
RuntimeType: "gateway",
InstanceMode: InstanceModeLite,
Status: "running",
WorkspacePath: &workspacePath,
}
service.syncInstance(context.Background(), instance)
if len(instanceRepo.updated) != 0 {
t.Fatalf("sync should not update V2 gateway instance via pod status, got %#v", instanceRepo.updated)
}
}
func TestV2RuntimeTypeForInstanceRequiresLiteGatewayBoundary(t *testing.T) {
workspacePath := "/workspaces/openclaw/user-45/instance-191"
lite := &models.Instance{
ID: 191,
Type: "openclaw",
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
WorkspacePath: &workspacePath,
}
if runtimeType, ok := v2RuntimeTypeForInstance(lite); !ok || runtimeType != RuntimeTypeOpenClaw {
t.Fatalf("lite gateway instance was not recognized: %q %v", runtimeType, ok)
}
pro := &models.Instance{
ID: 192,
Type: "openclaw",
RuntimeType: RuntimeBackendDesktop,
InstanceMode: InstanceModePro,
WorkspacePath: &workspacePath,
}
if runtimeType, ok := v2RuntimeTypeForInstance(pro); ok {
t.Fatalf("pro desktop instance must not be scheduler-managed, got %q", runtimeType)
}
}
func TestInstanceModeCapacityCanDisablePro(t *testing.T) {
t.Setenv("CLAWMANAGER_PRO_CAPACITY", "0")
service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()}
err := service.enforceInstanceModeLimits(context.Background(), InstanceModePro, 2, 4, 20, 0)
if err == nil || !strings.Contains(err.Error(), "pro instance mode is disabled") {
t.Fatalf("capacity error = %v, want pro disabled", err)
}
}
func TestInstanceModeResourceLimitRejectsOversizedPro(t *testing.T) {
t.Setenv("CLAWMANAGER_PRO_MAX_CPU_CORES", "1.5")
service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()}
err := service.enforceInstanceModeLimits(context.Background(), InstanceModePro, 2, 4, 20, 0)
if err == nil || !strings.Contains(err.Error(), "pro CPU cores exceed mode limit") {
t.Fatalf("resource limit error = %v, want pro CPU limit", err)
}
}
func TestInstanceModeResourceLimitAllowsOversizedLite(t *testing.T) {
t.Setenv("CLAWMANAGER_LITE_MAX_CPU_CORES", "1.5")
service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()}
err := service.enforceInstanceModeLimits(context.Background(), InstanceModeLite, 2, 4, 20, 0)
if err != nil {
t.Fatalf("resource limit error = %v, want lite to skip dedicated resource limits", err)
}
}
type v2LifecycleInstanceRepo struct {
byID map[int]*models.Instance
created []models.Instance
updated []models.Instance
deleted []int
workspacePath map[int]string
runtimeStates map[int]v2RuntimeStateRecord
nextID int
}
type v2RuntimeStateRecord struct {
status string
generation int
message *string
}
func newV2LifecycleInstanceRepo() *v2LifecycleInstanceRepo {
return &v2LifecycleInstanceRepo{
byID: map[int]*models.Instance{},
workspacePath: map[int]string{},
runtimeStates: map[int]v2RuntimeStateRecord{},
nextID: 1,
}
}
func (r *v2LifecycleInstanceRepo) Create(instance *models.Instance) error {
if instance.ID == 0 {
instance.ID = r.nextID
r.nextID++
}
copy := *instance
r.byID[instance.ID] = &copy
r.created = append(r.created, copy)
return nil
}
func (r *v2LifecycleInstanceRepo) GetByID(id int) (*models.Instance, error) {
return r.byID[id], nil
}
func (r *v2LifecycleInstanceRepo) GetByAccessToken(string) (*models.Instance, error) {
return nil, nil
}
func (r *v2LifecycleInstanceRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) {
return nil, nil
}
func (r *v2LifecycleInstanceRepo) GetAll(offset, limit int) ([]models.Instance, error) {
return nil, nil
}
func (r *v2LifecycleInstanceRepo) CountAll() (int, error) { return len(r.byID), nil }
func (r *v2LifecycleInstanceRepo) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) {
instances := make([]models.Instance, 0)
for _, instance := range r.byID {
if instance.UserID == userID {
instances = append(instances, *instance)
}
}
return instances, nil
}
func (r *v2LifecycleInstanceRepo) CountByUserID(userID int) (int, error) {
count := 0
for _, instance := range r.byID {
if instance.UserID == userID {
count++
}
}
return count, nil
}
func (r *v2LifecycleInstanceRepo) CountActiveByMode(ctx context.Context, mode string) (int, error) {
count := 0
for _, instance := range r.byID {
if instance.InstanceMode == mode && (instance.Status == "creating" || instance.Status == "running") {
count++
}
}
return count, nil
}
func (r *v2LifecycleInstanceRepo) ExistsByUserIDAndName(userID int, name string) (bool, error) {
normalized := strings.TrimSpace(strings.ToLower(name))
for _, instance := range r.byID {
if instance.UserID == userID && strings.TrimSpace(strings.ToLower(instance.Name)) == normalized {
return true, nil
}
}
return false, nil
}
func (r *v2LifecycleInstanceRepo) GetAllRunning() ([]models.Instance, error) {
instances := make([]models.Instance, 0, len(r.byID))
for _, instance := range r.byID {
instances = append(instances, *instance)
}
return instances, nil
}
func (r *v2LifecycleInstanceRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) {
return nil, nil
}
func (r *v2LifecycleInstanceRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) {
return nil, nil
}
func (r *v2LifecycleInstanceRepo) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error {
instance := r.byID[id]
if instance != nil {
instance.Status = status
instance.RuntimeGeneration = generation
instance.RuntimeErrorMessage = message
}
r.runtimeStates[id] = v2RuntimeStateRecord{status: status, generation: generation, message: message}
return nil
}
func (r *v2LifecycleInstanceRepo) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error {
instance := r.byID[id]
if instance != nil {
instance.WorkspacePath = &workspacePath
}
r.workspacePath[id] = workspacePath
return nil
}
func (r *v2LifecycleInstanceRepo) UpdateWorkspaceUsage(context.Context, int, int64) error {
return nil
}
func (r *v2LifecycleInstanceRepo) Update(instance *models.Instance) error {
copy := *instance
r.byID[instance.ID] = &copy
r.updated = append(r.updated, copy)
return nil
}
func (r *v2LifecycleInstanceRepo) Delete(id int) error {
delete(r.byID, id)
r.deleted = append(r.deleted, id)
return nil
}
type v2LifecycleQuotaRepo struct{}
func (v2LifecycleQuotaRepo) Create(*models.UserQuota) error { return nil }
func (v2LifecycleQuotaRepo) GetByUserID(userID int) (*models.UserQuota, error) {
return &models.UserQuota{
UserID: userID,
MaxInstances: 100,
MaxCPUCores: 100,
MaxMemoryGB: 1000,
MaxStorageGB: 1000,
MaxGPUCount: 8,
}, nil
}
func (v2LifecycleQuotaRepo) Update(*models.UserQuota) error { return nil }
func (v2LifecycleQuotaRepo) DeleteByUserID(int) error { return nil }
func (v2LifecycleQuotaRepo) CreateDefaultQuota(userID int) (*models.UserQuota, error) {
return v2LifecycleQuotaRepo{}.GetByUserID(userID)
}
type fixedV2QuotaRepo struct {
cpu float64
memory int
storage int
gpu int
}
func (fixedV2QuotaRepo) Create(*models.UserQuota) error { return nil }
func (r fixedV2QuotaRepo) GetByUserID(userID int) (*models.UserQuota, error) {
return &models.UserQuota{
UserID: userID,
MaxInstances: 100,
MaxCPUCores: r.cpu,
MaxMemoryGB: r.memory,
MaxStorageGB: r.storage,
MaxGPUCount: r.gpu,
}, nil
}
func (fixedV2QuotaRepo) Update(*models.UserQuota) error { return nil }
func (fixedV2QuotaRepo) DeleteByUserID(int) error { return nil }
func (r fixedV2QuotaRepo) CreateDefaultQuota(userID int) (*models.UserQuota, error) {
return r.GetByUserID(userID)
}
backend/internal/services/instance_visibility_test.go
+178 -159
View File
@@ -1,159 +1,178 @@
package services
import (
"testing"
"clawreef/internal/models"
)
// stubInstanceVisibilityRepo is a minimal InstanceRepository used by the
// visibility tests below. It only implements the read paths exercised by
// GetByUserID / GetAllInstances; every other method panics so that any
// accidental call surfaces as a test failure instead of a silent stub.
type stubInstanceVisibilityRepo struct {
all []models.Instance
}
func (r *stubInstanceVisibilityRepo) byUser(userID int) []models.Instance {
out := make([]models.Instance, 0, len(r.all))
for _, inst := range r.all {
if inst.UserID == userID {
out = append(out, inst)
}
}
return out
}
func (r *stubInstanceVisibilityRepo) Create(*models.Instance) error { panic("not used") }
func (r *stubInstanceVisibilityRepo) GetByID(int) (*models.Instance, error) {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) GetByAccessToken(string) (*models.Instance, error) {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) GetAll(offset, limit int) ([]models.Instance, error) {
return paginate(r.all, offset, limit), nil
}
func (r *stubInstanceVisibilityRepo) CountAll() (int, error) { return len(r.all), nil }
func (r *stubInstanceVisibilityRepo) GetByUserID(userID, offset, limit int) ([]models.Instance, error) {
return paginate(r.byUser(userID), offset, limit), nil
}
func (r *stubInstanceVisibilityRepo) CountByUserID(userID int) (int, error) {
return len(r.byUser(userID)), nil
}
func (r *stubInstanceVisibilityRepo) ExistsByUserIDAndName(int, string) (bool, error) {
return false, nil
}
func (r *stubInstanceVisibilityRepo) GetAllRunning() ([]models.Instance, error) {
return nil, nil
}
func (r *stubInstanceVisibilityRepo) Update(*models.Instance) error { panic("not used") }
func (r *stubInstanceVisibilityRepo) Delete(int) error { panic("not used") }
func paginate(items []models.Instance, offset, limit int) []models.Instance {
if offset >= len(items) {
return []models.Instance{}
}
end := offset + limit
if end > len(items) {
end = len(items)
}
out := make([]models.Instance, end-offset)
copy(out, items[offset:end])
return out
}
// TestGetByUserIDFiltersByCaller locks in the workspace-view contract: the
// caller-scoped listing must return only the caller's own instances,
// regardless of any role the caller may hold elsewhere. Regression guard
// for the admin-cross-user-leakage bug in which role=admin widened this
// endpoint to every user's instances.
func TestGetByUserIDFiltersByCaller(t *testing.T) {
t.Parallel()
repo := &stubInstanceVisibilityRepo{
all: []models.Instance{
{ID: 1, UserID: 10, Name: "alice-1"},
{ID: 2, UserID: 10, Name: "alice-2"},
{ID: 3, UserID: 20, Name: "bob-1"},
},
}
svc := &instanceService{instanceRepo: repo}
instances, total, err := svc.GetByUserID(10, 0, 50)
if err != nil {
t.Fatalf("GetByUserID returned error: %v", err)
}
if total != 2 {
t.Fatalf("expected total=2 for alice, got %d", total)
}
if len(instances) != 2 || instances[0].Name != "alice-1" || instances[1].Name != "alice-2" {
t.Fatalf("expected alice's instances only, got %+v", instances)
}
instances, total, err = svc.GetByUserID(20, 0, 50)
if err != nil {
t.Fatalf("GetByUserID returned error: %v", err)
}
if total != 1 {
t.Fatalf("expected total=1 for bob, got %d", total)
}
if len(instances) != 1 || instances[0].Name != "bob-1" {
t.Fatalf("expected bob's instance only, got %+v", instances)
}
// A user with no instances sees an empty list, never someone else's.
instances, total, err = svc.GetByUserID(99, 0, 50)
if err != nil {
t.Fatalf("GetByUserID returned error: %v", err)
}
if total != 0 || len(instances) != 0 {
t.Fatalf("expected empty listing for user 99, got total=%d instances=%+v", total, instances)
}
}
// TestGetAllInstancesReturnsEveryUser verifies the admin-console surface:
// cross-user listing, with pagination, independent of caller identity (the
// admin middleware gates reachability at the route layer, not here).
func TestGetAllInstancesReturnsEveryUser(t *testing.T) {
t.Parallel()
repo := &stubInstanceVisibilityRepo{
all: []models.Instance{
{ID: 1, UserID: 10, Name: "alice-1"},
{ID: 2, UserID: 10, Name: "alice-2"},
{ID: 3, UserID: 20, Name: "bob-1"},
},
}
svc := &instanceService{instanceRepo: repo}
instances, total, err := svc.GetAllInstances(0, 50)
if err != nil {
t.Fatalf("GetAllInstances returned error: %v", err)
}
if total != 3 {
t.Fatalf("expected total=3 across users, got %d", total)
}
if len(instances) != 3 {
t.Fatalf("expected 3 instances, got %d", len(instances))
}
// Pagination still respected.
page1, _, err := svc.GetAllInstances(0, 2)
if err != nil {
t.Fatalf("GetAllInstances page1 error: %v", err)
}
if len(page1) != 2 {
t.Fatalf("expected page1 size=2, got %d", len(page1))
}
page2, _, err := svc.GetAllInstances(2, 2)
if err != nil {
t.Fatalf("GetAllInstances page2 error: %v", err)
}
if len(page2) != 1 || page2[0].Name != "bob-1" {
t.Fatalf("expected page2=[bob-1], got %+v", page2)
}
}
package services
import (
"context"
"testing"
"clawreef/internal/models"
)
// stubInstanceVisibilityRepo is a minimal InstanceRepository used by the
// visibility tests below. It only implements the read paths exercised by
// GetByUserID / GetAllInstances; every other method panics so that any
// accidental call surfaces as a test failure instead of a silent stub.
type stubInstanceVisibilityRepo struct {
all []models.Instance
}
func (r *stubInstanceVisibilityRepo) byUser(userID int) []models.Instance {
out := make([]models.Instance, 0, len(r.all))
for _, inst := range r.all {
if inst.UserID == userID {
out = append(out, inst)
}
}
return out
}
func (r *stubInstanceVisibilityRepo) Create(*models.Instance) error { panic("not used") }
func (r *stubInstanceVisibilityRepo) GetByID(int) (*models.Instance, error) {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) GetByAccessToken(string) (*models.Instance, error) {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) GetAll(offset, limit int) ([]models.Instance, error) {
return paginate(r.all, offset, limit), nil
}
func (r *stubInstanceVisibilityRepo) CountAll() (int, error) { return len(r.all), nil }
func (r *stubInstanceVisibilityRepo) GetByUserID(userID, offset, limit int) ([]models.Instance, error) {
return paginate(r.byUser(userID), offset, limit), nil
}
func (r *stubInstanceVisibilityRepo) CountByUserID(userID int) (int, error) {
return len(r.byUser(userID)), nil
}
func (r *stubInstanceVisibilityRepo) CountActiveByMode(context.Context, string) (int, error) {
return 0, nil
}
func (r *stubInstanceVisibilityRepo) ExistsByUserIDAndName(int, string) (bool, error) {
return false, nil
}
func (r *stubInstanceVisibilityRepo) GetAllRunning() ([]models.Instance, error) {
return nil, nil
}
func (r *stubInstanceVisibilityRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) UpdateRuntimeState(context.Context, int, string, int, *string) error {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) SetWorkspacePath(context.Context, int, string) error {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) UpdateWorkspaceUsage(context.Context, int, int64) error {
panic("not used")
}
func (r *stubInstanceVisibilityRepo) Update(*models.Instance) error { panic("not used") }
func (r *stubInstanceVisibilityRepo) Delete(int) error { panic("not used") }
func paginate(items []models.Instance, offset, limit int) []models.Instance {
if offset >= len(items) {
return []models.Instance{}
}
end := offset + limit
if end > len(items) {
end = len(items)
}
out := make([]models.Instance, end-offset)
copy(out, items[offset:end])
return out
}
// TestGetByUserIDFiltersByCaller locks in the workspace-view contract: the
// caller-scoped listing must return only the caller's own instances,
// regardless of any role the caller may hold elsewhere. Regression guard
// for the admin-cross-user-leakage bug in which role=admin widened this
// endpoint to every user's instances.
func TestGetByUserIDFiltersByCaller(t *testing.T) {
t.Parallel()
repo := &stubInstanceVisibilityRepo{
all: []models.Instance{
{ID: 1, UserID: 10, Name: "alice-1"},
{ID: 2, UserID: 10, Name: "alice-2"},
{ID: 3, UserID: 20, Name: "bob-1"},
},
}
svc := &instanceService{instanceRepo: repo}
instances, total, err := svc.GetByUserID(10, 0, 50)
if err != nil {
t.Fatalf("GetByUserID returned error: %v", err)
}
if total != 2 {
t.Fatalf("expected total=2 for alice, got %d", total)
}
if len(instances) != 2 || instances[0].Name != "alice-1" || instances[1].Name != "alice-2" {
t.Fatalf("expected alice's instances only, got %+v", instances)
}
instances, total, err = svc.GetByUserID(20, 0, 50)
if err != nil {
t.Fatalf("GetByUserID returned error: %v", err)
}
if total != 1 {
t.Fatalf("expected total=1 for bob, got %d", total)
}
if len(instances) != 1 || instances[0].Name != "bob-1" {
t.Fatalf("expected bob's instance only, got %+v", instances)
}
// A user with no instances sees an empty list, never someone else's.
instances, total, err = svc.GetByUserID(99, 0, 50)
if err != nil {
t.Fatalf("GetByUserID returned error: %v", err)
}
if total != 0 || len(instances) != 0 {
t.Fatalf("expected empty listing for user 99, got total=%d instances=%+v", total, instances)
}
}
// TestGetAllInstancesReturnsEveryUser verifies the admin-console surface:
// cross-user listing, with pagination, independent of caller identity (the
// admin middleware gates reachability at the route layer, not here).
func TestGetAllInstancesReturnsEveryUser(t *testing.T) {
t.Parallel()
repo := &stubInstanceVisibilityRepo{
all: []models.Instance{
{ID: 1, UserID: 10, Name: "alice-1"},
{ID: 2, UserID: 10, Name: "alice-2"},
{ID: 3, UserID: 20, Name: "bob-1"},
},
}
svc := &instanceService{instanceRepo: repo}
instances, total, err := svc.GetAllInstances(0, 50)
if err != nil {
t.Fatalf("GetAllInstances returned error: %v", err)
}
if total != 3 {
t.Fatalf("expected total=3 across users, got %d", total)
}
if len(instances) != 3 {
t.Fatalf("expected 3 instances, got %d", len(instances))
}
// Pagination still respected.
page1, _, err := svc.GetAllInstances(0, 2)
if err != nil {
t.Fatalf("GetAllInstances page1 error: %v", err)
}
if len(page1) != 2 {
t.Fatalf("expected page1 size=2, got %d", len(page1))
}
page2, _, err := svc.GetAllInstances(2, 2)
if err != nil {
t.Fatalf("GetAllInstances page2 error: %v", err)
}
if len(page2) != 1 || page2[0].Name != "bob-1" {
t.Fatalf("expected page2=[bob-1], got %+v", page2)
}
}
backend/internal/services/k8s/client.go
+18 -12
View File
@@ -29,12 +29,15 @@ const (
// Client wraps the Kubernetes client
type Client struct {
Clientset kubernetes.Interface
Config *rest.Config
Namespace string
StorageClass string
HostPathPrefix string
Mode ConnectionMode
Clientset kubernetes.Interface
Config *rest.Config
Namespace string
StorageClass string
HostPathPrefix string
WorkspaceRoot string
WorkspaceNFSServer string
WorkspaceNFSPath string
Mode ConnectionMode
}
var (
@@ -97,12 +100,15 @@ func Initialize(cfg *config.Config) error {
}
globalClient = &Client{
Clientset: clientset,
Config: restConfig,
Namespace: cfg.GetNamespace(),
StorageClass: cfg.GetStorageClass(),
HostPathPrefix: cfg.GetHostPathPrefix(),
Mode: detectedMode,
Clientset: clientset,
Config: restConfig,
Namespace: cfg.GetNamespace(),
StorageClass: cfg.GetStorageClass(),
HostPathPrefix: cfg.GetHostPathPrefix(),
WorkspaceRoot: cfg.Runtime.WorkspaceRoot,
WorkspaceNFSServer: cfg.Runtime.WorkspaceNFSServer,
WorkspaceNFSPath: cfg.Runtime.WorkspaceNFSPath,
Mode: detectedMode,
}
return nil
backend/internal/services/k8s/instance_deployment_service.go
+376
View File
@@ -0,0 +1,376 @@
package k8s
import (
"context"
"encoding/json"
"fmt"
"reflect"
"time"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
const instanceDeploymentRestartedAtAnnotation = "clawmanager.io/restarted-at"
type InstanceDeploymentService struct {
client *Client
namespaceService *NamespaceService
}
func NewInstanceDeploymentService() *InstanceDeploymentService {
return &InstanceDeploymentService{
client: globalClient,
namespaceService: NewNamespaceService(),
}
}
func InstanceDeploymentName(instanceID int) string {
return sanitizeK8sName(fmt.Sprintf("clawreef-%d-deployment", instanceID))
}
func (s *InstanceDeploymentService) EnsureDeployment(ctx context.Context, config PodConfig, replicas int32) (*appsv1.Deployment, error) {
if s.client == nil {
return nil, fmt.Errorf("k8s client not initialized")
}
if _, err := s.namespaceService.EnsureNamespace(ctx, config.UserID); err != nil {
return nil, fmt.Errorf("failed to ensure namespace: %w", err)
}
desired := BuildInstanceDeployment(s.client, config, replicas)
deployments := s.client.Clientset.AppsV1().Deployments(desired.Namespace)
existing, err := deployments.Get(ctx, desired.Name, metav1.GetOptions{})
if errors.IsNotFound(err) {
created, createErr := deployments.Create(ctx, desired, metav1.CreateOptions{})
if createErr != nil {
return nil, fmt.Errorf("failed to create instance deployment %s/%s: %w", desired.Namespace, desired.Name, createErr)
}
return created, nil
}
if err != nil {
return nil, fmt.Errorf("failed to get instance deployment %s/%s: %w", desired.Namespace, desired.Name, err)
}
if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) {
return nil, fmt.Errorf("instance deployment %s/%s selector mismatch; delete and recreate the deployment to change immutable selector", desired.Namespace, desired.Name)
}
updated := existing.DeepCopy()
if updated.Labels == nil {
updated.Labels = map[string]string{}
}
for key, value := range desired.Labels {
updated.Labels[key] = value
}
updated.Spec.Replicas = desired.Spec.Replicas
updated.Spec.Template = desired.Spec.Template
result, updateErr := deployments.Update(ctx, updated, metav1.UpdateOptions{})
if updateErr != nil {
return nil, fmt.Errorf("failed to update instance deployment %s/%s: %w", desired.Namespace, desired.Name, updateErr)
}
return result, nil
}
func (s *InstanceDeploymentService) ScaleDeployment(ctx context.Context, userID, instanceID int, replicas int32) error {
if s.client == nil {
return fmt.Errorf("k8s client not initialized")
}
namespace := s.client.GetNamespace(userID)
name := InstanceDeploymentName(instanceID)
patch, err := json.Marshal(map[string]any{
"spec": map[string]any{"replicas": replicas},
})
if err != nil {
return fmt.Errorf("failed to build instance deployment scale patch: %w", err)
}
if _, err := s.client.Clientset.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil {
return fmt.Errorf("failed to scale instance deployment %s/%s: %w", namespace, name, err)
}
return nil
}
func (s *InstanceDeploymentService) RestartDeployment(ctx context.Context, userID, instanceID int) error {
if s.client == nil {
return fmt.Errorf("k8s client not initialized")
}
namespace := s.client.GetNamespace(userID)
name := InstanceDeploymentName(instanceID)
patch, err := json.Marshal(map[string]any{
"spec": map[string]any{
"template": map[string]any{
"metadata": map[string]any{
"annotations": map[string]string{
instanceDeploymentRestartedAtAnnotation: time.Now().UTC().Format(time.RFC3339Nano),
},
},
},
},
})
if err != nil {
return fmt.Errorf("failed to build instance deployment restart patch: %w", err)
}
if _, err := s.client.Clientset.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil {
return fmt.Errorf("failed to restart instance deployment %s/%s: %w", namespace, name, err)
}
return nil
}
func (s *InstanceDeploymentService) DeleteDeployment(ctx context.Context, userID, instanceID int) error {
if s.client == nil {
return fmt.Errorf("k8s client not initialized")
}
namespace := s.client.GetNamespace(userID)
name := InstanceDeploymentName(instanceID)
err := s.client.Clientset.AppsV1().Deployments(namespace).Delete(ctx, name, metav1.DeleteOptions{})
if err != nil && !errors.IsNotFound(err) {
return fmt.Errorf("failed to delete instance deployment %s/%s: %w", namespace, name, err)
}
return nil
}
func (s *InstanceDeploymentService) GetDeployment(ctx context.Context, userID, instanceID int) (*appsv1.Deployment, error) {
if s.client == nil {
return nil, fmt.Errorf("k8s client not initialized")
}
namespace := s.client.GetNamespace(userID)
name := InstanceDeploymentName(instanceID)
deployment, err := s.client.Clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get instance deployment %s/%s: %w", namespace, name, err)
}
return deployment, nil
}
func (s *InstanceDeploymentService) GetActivePod(ctx context.Context, userID, instanceID int) (*corev1.Pod, error) {
if s.client == nil {
return nil, fmt.Errorf("k8s client not initialized")
}
namespace := s.client.GetNamespace(userID)
selector := fmt.Sprintf("instance-id=%d,app=clawreef", instanceID)
pods, err := s.client.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector})
if err != nil {
return nil, fmt.Errorf("failed to list instance deployment pods: %w", err)
}
if len(pods.Items) == 0 {
return nil, fmt.Errorf("pod not found for instance %d", instanceID)
}
for _, pod := range pods.Items {
if pod.DeletionTimestamp == nil && pod.Status.Phase == corev1.PodRunning && podReady(&pod) {
selected := pod
return &selected, nil
}
}
selected := pods.Items[0]
return &selected, nil
}
func BuildInstanceDeployment(client *Client, config PodConfig, replicas int32) *appsv1.Deployment {
namespace := client.GetNamespace(config.UserID)
name := InstanceDeploymentName(config.InstanceID)
runtimeType := normalizePodRuntimeType(config.RuntimeType)
labels := map[string]string{
"app": "clawreef",
"instance-id": fmt.Sprintf("%d", config.InstanceID),
"instance-name": config.InstanceName,
"user-id": fmt.Sprintf("%d", config.UserID),
"instance-type": config.Type,
"runtime-type": runtimeType,
"managed-by": "clawreef",
}
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: copyStringMap(labels),
},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "clawreef",
"instance-id": fmt.Sprintf("%d", config.InstanceID),
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: instanceDeploymentAnnotations(config),
Labels: labels,
},
Spec: buildInstanceDeploymentPodSpec(client, config, runtimeType),
},
},
}
}
func instanceDeploymentAnnotations(config PodConfig) map[string]string {
if config.SecurityMode == PodSecurityChromiumCompat {
return map[string]string{"container.apparmor.security.beta.kubernetes.io/desktop": "unconfined"}
}
return nil
}
func buildInstanceDeploymentPodSpec(client *Client, config PodConfig, runtimeType string) corev1.PodSpec {
pvcName := client.GetPVCName(config.InstanceID)
if config.ContainerPort == 0 {
config.ContainerPort = 3001
}
pullPolicy := config.ImagePullPolicy
if pullPolicy == "" {
pullPolicy = corev1.PullIfNotPresent
}
resources := corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)),
corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)),
corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)),
},
}
if config.GPUEnabled && config.GPUCount > 0 {
resources.Limits["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount))
resources.Requests["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount))
}
container := corev1.Container{
Name: "desktop",
Image: config.Image,
ImagePullPolicy: pullPolicy,
Ports: []corev1.ContainerPort{{
ContainerPort: config.ContainerPort,
Name: "http",
}},
StartupProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}},
FailureThreshold: 30,
PeriodSeconds: 5,
TimeoutSeconds: 2,
},
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}},
InitialDelaySeconds: 3,
PeriodSeconds: 5,
TimeoutSeconds: 2,
FailureThreshold: 6,
},
LivenessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}},
InitialDelaySeconds: 15,
PeriodSeconds: 10,
TimeoutSeconds: 2,
FailureThreshold: 3,
},
SecurityContext: buildContainerSecurityContext(config.SecurityMode),
Resources: resources,
VolumeMounts: []corev1.VolumeMount{{
Name: "data",
MountPath: config.MountPath,
}},
Env: []corev1.EnvVar{
{Name: "INSTANCE_ID", Value: fmt.Sprintf("%d", config.InstanceID)},
{Name: "USER_ID", Value: fmt.Sprintf("%d", config.UserID)},
},
}
if runtimeType == "shell" {
container.Ports = nil
container.StartupProbe = nil
container.ReadinessProbe = nil
container.LivenessProbe = nil
}
for key, value := range config.ExtraEnv {
container.Env = append(container.Env, corev1.EnvVar{Name: key, Value: value})
}
for _, secretName := range config.EnvFromSecretNames {
if secretName == "" {
continue
}
container.EnvFrom = append(container.EnvFrom, corev1.EnvFromSource{
SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: secretName}},
})
}
spec := corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyAlways,
SecurityContext: buildPodSecurityContext(config.FSGroup),
Containers: []corev1.Container{container},
Volumes: []corev1.Volume{{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName},
},
}},
}
for _, mount := range config.ExtraPVCMounts {
if mount.Name == "" || mount.ClaimName == "" || mount.MountPath == "" {
continue
}
spec.Volumes = append(spec.Volumes, corev1.Volume{
Name: mount.Name,
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: mount.ClaimName, ReadOnly: mount.ReadOnly},
},
})
spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: mount.Name,
MountPath: mount.MountPath,
ReadOnly: mount.ReadOnly,
})
}
for index, fix := range config.VolumeOwnershipFixes {
if fix.Name == "" || fix.MountPath == "" || fix.UID < 0 || fix.GID < 0 {
continue
}
spec.InitContainers = append(spec.InitContainers, buildVolumeOwnershipInitContainer(index, config.Image, pullPolicy, fix))
}
for _, mount := range config.ConfigMapFileMounts {
if mount.Name == "" || mount.ConfigMapName == "" || mount.Key == "" || mount.MountPath == "" {
continue
}
spec.Volumes = append(spec.Volumes, corev1.Volume{
Name: mount.Name,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{Name: mount.ConfigMapName},
Items: []corev1.KeyToPath{{Key: mount.Key, Path: mount.Key}},
},
},
})
volumeMount := corev1.VolumeMount{Name: mount.Name, MountPath: mount.MountPath, ReadOnly: true}
if !mount.AsDirectory {
volumeMount.SubPath = mount.Key
}
spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, volumeMount)
}
if config.SHMSizeGB > 0 {
shmLimit := resource.MustParse(fmt.Sprintf("%dGi", config.SHMSizeGB))
spec.Volumes = append(spec.Volumes, corev1.Volume{
Name: "shm",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: &shmLimit},
},
})
spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shm", MountPath: "/dev/shm"})
}
return spec
}
func podReady(pod *corev1.Pod) bool {
for _, condition := range pod.Status.Conditions {
if condition.Type == corev1.PodReady {
return condition.Status == corev1.ConditionTrue
}
}
return false
}
backend/internal/services/k8s/instance_deployment_service_test.go
+99
View File
@@ -0,0 +1,99 @@
package k8s
import (
"context"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
func TestBuildInstanceDeploymentUsesStableIdentityAndPVC(t *testing.T) {
client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"}
deployment := BuildInstanceDeployment(client, PodConfig{
InstanceID: 42,
InstanceName: "Pro Desktop",
UserID: 7,
Type: "openclaw",
RuntimeType: "desktop",
CPUCores: 2,
MemoryGB: 4,
Image: "registry/openclaw:pro",
MountPath: "/config",
ContainerPort: 3001,
ImagePullPolicy: corev1.PullIfNotPresent,
}, 1)
if deployment.Name != "clawreef-42-deployment" {
t.Fatalf("deployment name = %q, want stable instance deployment name", deployment.Name)
}
if deployment.Namespace != "clawreef-user-7" {
t.Fatalf("namespace = %q, want user namespace", deployment.Namespace)
}
if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 {
t.Fatalf("replicas = %#v, want 1", deployment.Spec.Replicas)
}
if got := deployment.Spec.Selector.MatchLabels["instance-id"]; got != "42" {
t.Fatalf("selector instance-id = %q, want 42", got)
}
template := deployment.Spec.Template
if got := template.Labels["runtime-type"]; got != "desktop" {
t.Fatalf("template runtime-type = %q, want desktop", got)
}
container := template.Spec.Containers[0]
if container.Name != "desktop" || container.Image != "registry/openclaw:pro" {
t.Fatalf("unexpected container: %#v", container)
}
if template.Spec.RestartPolicy != corev1.RestartPolicyAlways {
t.Fatalf("restart policy = %q, want Always", template.Spec.RestartPolicy)
}
if len(template.Spec.Volumes) == 0 || template.Spec.Volumes[0].PersistentVolumeClaim == nil {
t.Fatalf("expected PVC data volume, got %#v", template.Spec.Volumes)
}
if got := template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName; got != "clawreef-42-pvc" {
t.Fatalf("PVC name = %q, want clawreef-42-pvc", got)
}
}
func TestInstanceDeploymentServiceEnsureAndScale(t *testing.T) {
client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"}
service := &InstanceDeploymentService{
client: client,
namespaceService: &NamespaceService{client: client},
}
config := PodConfig{
InstanceID: 43,
InstanceName: "Pro Desktop",
UserID: 8,
Type: "openclaw",
RuntimeType: "desktop",
CPUCores: 2,
MemoryGB: 4,
Image: "registry/openclaw:v1",
MountPath: "/config",
ContainerPort: 3001,
}
if _, err := service.EnsureDeployment(context.Background(), config, 1); err != nil {
t.Fatalf("EnsureDeployment create returned error: %v", err)
}
deployment, err := client.Clientset.AppsV1().Deployments("clawreef-user-8").Get(context.Background(), "clawreef-43-deployment", metav1.GetOptions{})
if err != nil {
t.Fatalf("get deployment: %v", err)
}
if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 {
t.Fatalf("created replicas = %#v, want 1", deployment.Spec.Replicas)
}
if err := service.ScaleDeployment(context.Background(), 8, 43, 0); err != nil {
t.Fatalf("ScaleDeployment returned error: %v", err)
}
scaled, err := client.Clientset.AppsV1().Deployments("clawreef-user-8").Get(context.Background(), "clawreef-43-deployment", metav1.GetOptions{})
if err != nil {
t.Fatalf("get scaled deployment: %v", err)
}
if scaled.Spec.Replicas == nil || *scaled.Spec.Replicas != 0 {
t.Fatalf("scaled replicas = %#v, want 0", scaled.Spec.Replicas)
}
}
backend/internal/services/k8s/pvc_service.go
+207 -5
View File
@@ -3,6 +3,10 @@ package k8s
import (
"context"
"fmt"
"net"
"os"
"path"
"path/filepath"
"sort"
"strings"
"time"
@@ -136,6 +140,12 @@ func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, st
storageClass = s.client.StorageClass
}
storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB))
useWorkspaceNFS := strings.TrimSpace(s.client.WorkspaceNFSServer) != ""
if useWorkspaceNFS {
if err := s.ensureTeamSharedWorkspaceDirectory(userID, teamID); err != nil {
return nil, err
}
}
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
@@ -165,6 +175,11 @@ func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, st
if errors.IsAlreadyExists(err) {
existingPVC, getErr := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
if getErr == nil && existingPVC != nil && existingPVC.Labels["team-id"] == fmt.Sprintf("%d", teamID) {
if useWorkspaceNFS {
if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, existingPVC, userID, teamID, storageSizeGB, storageClass, fmt.Sprintf("clawreef-pv-user-%d-team-%d-shared", userID, teamID)); err != nil {
return nil, err
}
}
if existingPVC.Status.Phase != corev1.ClaimBound {
go s.monitorTeamSharedPVCBinding(context.Background(), namespace, pvcName, userID, teamID, storageSizeGB, storageClass, 15*time.Second)
}
@@ -174,6 +189,11 @@ func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, st
return nil, fmt.Errorf("failed to create Team shared PVC %s: %w", pvcName, err)
}
if useWorkspaceNFS {
if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, createdPVC, userID, teamID, storageSizeGB, storageClass, fmt.Sprintf("clawreef-pv-user-%d-team-%d-shared", userID, teamID)); err != nil {
return nil, err
}
}
go s.monitorTeamSharedPVCBinding(context.Background(), namespace, pvcName, userID, teamID, storageSizeGB, storageClass, 15*time.Second)
return createdPVC, nil
}
@@ -218,11 +238,6 @@ func (s *PVCService) waitForTeamSharedPVCBinding(ctx context.Context, namespace,
func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, userID, teamID, storageSizeGB int, storageClass string) (*corev1.PersistentVolumeClaim, error) {
pvName := fmt.Sprintf("clawreef-pv-user-%d-team-%d-shared", userID, teamID)
hostPathPrefix := "/data/clawreef"
if s.client != nil && s.client.HostPathPrefix != "" {
hostPathPrefix = s.client.HostPathPrefix
}
hostPath := fmt.Sprintf("%s/user-%d/team-%d-shared", hostPathPrefix, userID, teamID)
existingPV, err := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
if err == nil && existingPV != nil {
@@ -245,6 +260,16 @@ func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pv
if err != nil {
return nil, fmt.Errorf("failed to get Team shared PVC %s for UID: %w", pvcName, err)
}
if strings.TrimSpace(s.client.WorkspaceNFSServer) != "" {
return s.createWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, pvc, userID, teamID, storageSizeGB, storageClass, pvName)
}
hostPathPrefix := "/data/clawreef"
if s.client != nil && s.client.HostPathPrefix != "" {
hostPathPrefix = s.client.HostPathPrefix
}
hostPath := fmt.Sprintf("%s/user-%d/team-%d-shared", hostPathPrefix, userID, teamID)
storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB))
nodeAffinity, err := s.hostPathPVNodeAffinity(ctx)
if err != nil {
@@ -305,6 +330,183 @@ func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pv
return pvc, nil
}
func (s *PVCService) createWorkspaceNFSPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, pvc *corev1.PersistentVolumeClaim, userID, teamID, storageSizeGB int, storageClass, pvName string) (*corev1.PersistentVolumeClaim, error) {
if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, pvc, userID, teamID, storageSizeGB, storageClass, pvName); err != nil {
return nil, err
}
time.Sleep(3 * time.Second)
pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get Team shared PVC after NFS PV creation: %w", err)
}
if pvc.Status.Phase != corev1.ClaimBound {
return nil, fmt.Errorf("Team shared PVC %s is still not bound after NFS PV creation, status: %s", pvcName, pvc.Status.Phase)
}
fmt.Printf("Team shared PVC %s successfully bound to NFS PV %s (%s:%s)\n", pvcName, pvName, s.client.WorkspaceNFSServer, teamSharedWorkspaceNFSPath(s.client.WorkspaceNFSPath, userID, teamID))
return pvc, nil
}
func (s *PVCService) ensureWorkspaceNFSPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, pvc *corev1.PersistentVolumeClaim, userID, teamID, storageSizeGB int, storageClass, pvName string) error {
if err := s.ensureTeamSharedWorkspaceDirectory(userID, teamID); err != nil {
return err
}
storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB))
nfsPath := teamSharedWorkspaceNFSPath(s.client.WorkspaceNFSPath, userID, teamID)
nfsServer, err := s.workspaceNFSServerForPV(ctx)
if err != nil {
return err
}
pv := &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: pvName,
Labels: map[string]string{
"app": "clawreef",
"user-id": fmt.Sprintf("%d", userID),
"team-id": fmt.Sprintf("%d", teamID),
"managed-by": "clawreef",
},
},
Spec: corev1.PersistentVolumeSpec{
Capacity: corev1.ResourceList{
corev1.ResourceStorage: storageSize,
},
AccessModes: []corev1.PersistentVolumeAccessMode{
corev1.ReadWriteMany,
},
PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain,
StorageClassName: storageClass,
PersistentVolumeSource: corev1.PersistentVolumeSource{
NFS: &corev1.NFSVolumeSource{
Server: nfsServer,
Path: nfsPath,
},
},
ClaimRef: &corev1.ObjectReference{
Kind: "PersistentVolumeClaim",
APIVersion: "v1",
Namespace: namespace,
Name: pvcName,
UID: pvc.UID,
ResourceVersion: pvc.ResourceVersion,
},
},
}
if _, err := s.client.Clientset.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil {
if errors.IsAlreadyExists(err) {
existing, getErr := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
if getErr != nil {
return fmt.Errorf("failed to get existing Team shared NFS PV %s: %w", pvName, getErr)
}
if existing.Spec.ClaimRef != nil && existing.Spec.ClaimRef.Namespace == namespace && existing.Spec.ClaimRef.Name == pvcName {
if existing.Spec.NFS != nil &&
strings.TrimSpace(existing.Spec.NFS.Server) == nfsServer &&
strings.TrimSpace(existing.Spec.NFS.Path) == nfsPath {
return nil
}
return fmt.Errorf("Team shared PV %s already exists for %s/%s but is not workspace NFS-backed", pvName, namespace, pvcName)
}
}
return fmt.Errorf("failed to create Team shared NFS PV %s: %w", pvName, err)
}
return nil
}
func (s *PVCService) workspaceNFSServerForPV(ctx context.Context) (string, error) {
server := strings.TrimSpace(s.client.WorkspaceNFSServer)
if server == "" || s.client.Clientset == nil {
return server, nil
}
serviceName, namespace, ok := workspaceNFSServiceRef(server, s.client.Namespace)
if !ok {
return server, nil
}
svc, err := s.client.Clientset.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return server, nil
}
return "", fmt.Errorf("failed to resolve workspace NFS service %s/%s: %w", namespace, serviceName, err)
}
clusterIP := strings.TrimSpace(svc.Spec.ClusterIP)
if clusterIP == "" || strings.EqualFold(clusterIP, corev1.ClusterIPNone) {
return server, nil
}
return clusterIP, nil
}
func workspaceNFSServiceRef(server, defaultNamespace string) (string, string, bool) {
host := strings.TrimSuffix(strings.TrimSpace(server), ".")
if host == "" || net.ParseIP(host) != nil {
return "", "", false
}
parts := strings.Split(strings.ToLower(host), ".")
if len(parts) == 1 {
namespace := strings.TrimSpace(defaultNamespace)
if namespace == "" {
namespace = "default"
}
return parts[0], namespace, true
}
if len(parts) >= 3 && parts[2] == "svc" && parts[0] != "" && parts[1] != "" {
return parts[0], parts[1], true
}
return "", "", false
}
func (s *PVCService) ensureTeamSharedWorkspaceDirectory(userID, teamID int) error {
root := strings.TrimSpace(s.client.WorkspaceRoot)
if root == "" {
return nil
}
dir := filepath.Join(root, filepath.FromSlash(TeamSharedWorkspaceRelativePath(userID, teamID)))
dirs := append([]string{dir}, teamSharedRuntimeSubdirectories(dir)...)
for _, target := range dirs {
if err := os.MkdirAll(target, 0o2775); err != nil {
return fmt.Errorf("failed to create Team shared runtime workspace directory %s: %w", target, err)
}
_ = os.Chown(target, 1000, 1000)
if err := os.Chmod(target, 0o2775); err != nil {
return fmt.Errorf("failed to chmod Team shared runtime workspace directory %s: %w", target, err)
}
}
return nil
}
func teamSharedRuntimeSubdirectories(root string) []string {
return []string{
filepath.Join(root, "status"),
filepath.Join(root, "inbox"),
filepath.Join(root, "results"),
filepath.Join(root, "tasks"),
}
}
func TeamSharedWorkspaceRelativePath(userID, teamID int) string {
return fmt.Sprintf("teams/user-%d/team-%d-shared", userID, teamID)
}
func TeamSharedWorkspacePath(workspaceRoot string, userID, teamID int) string {
root := strings.TrimRight(strings.TrimSpace(workspaceRoot), "/")
if root == "" {
root = "/workspaces"
}
return root + "/" + TeamSharedWorkspaceRelativePath(userID, teamID)
}
func teamSharedWorkspaceNFSPath(basePath string, userID, teamID int) string {
basePath = strings.TrimSpace(basePath)
if basePath == "" {
basePath = "/"
}
relativePath := TeamSharedWorkspaceRelativePath(userID, teamID)
if basePath == "/" {
return "/" + relativePath
}
return path.Join(basePath, relativePath)
}
func (s *PVCService) hostPathPVNodeAffinity(ctx context.Context) (*corev1.VolumeNodeAffinity, error) {
if s == nil || s.client == nil {
return nil, fmt.Errorf("k8s client not initialized")
backend/internal/services/k8s/pvc_service_test.go
+220
View File
@@ -2,10 +2,14 @@ package k8s
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/fake"
)
@@ -105,6 +109,222 @@ func TestHostPathPVNodeAffinitySkipsUnschedulableAndNotReadyNodes(t *testing.T)
requireHostnameAffinity(t, affinity, "node-c-host")
}
func TestCreatePVForTeamSharedPVCUsesWorkspaceNFSWhenConfigured(t *testing.T) {
ctx := context.Background()
workspaceRoot := t.TempDir()
namespace := "clawmanager-user-1"
pvcName := "clawreef-team-28-shared"
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: pvcName,
Namespace: namespace,
UID: types.UID("pvc-uid"),
ResourceVersion: "1",
},
Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound},
}
clientset := fake.NewSimpleClientset(pvc)
service := &PVCService{
client: &Client{
Clientset: clientset,
WorkspaceRoot: workspaceRoot,
WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
WorkspaceNFSPath: "/",
},
}
if _, err := service.createPVForTeamSharedPVC(ctx, namespace, pvcName, 1, 28, 10, "manual"); err != nil {
t.Fatalf("createPVForTeamSharedPVC returned error: %v", err)
}
pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-user-1-team-28-shared", metav1.GetOptions{})
if err != nil {
t.Fatalf("expected Team shared PV: %v", err)
}
if pv.Spec.PersistentVolumeSource.NFS == nil {
t.Fatalf("expected NFS PV source, got %#v", pv.Spec.PersistentVolumeSource)
}
if pv.Spec.PersistentVolumeSource.NFS.Server != "workspace-store.clawmanager-system.svc.cluster.local" {
t.Fatalf("unexpected NFS server: %#v", pv.Spec.PersistentVolumeSource.NFS)
}
if pv.Spec.PersistentVolumeSource.NFS.Path != "/teams/user-1/team-28-shared" {
t.Fatalf("unexpected NFS path: %#v", pv.Spec.PersistentVolumeSource.NFS)
}
if pv.Spec.NodeAffinity != nil {
t.Fatalf("NFS-backed Team PV must not pin runtime sharing to one node: %#v", pv.Spec.NodeAffinity)
}
if _, err := os.Stat(filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared")); err != nil {
t.Fatalf("expected runtime workspace team directory to be created: %v", err)
}
}
func TestCreatePVForTeamSharedPVCResolvesWorkspaceServiceDNS(t *testing.T) {
ctx := context.Background()
namespace := "clawmanager-user-1"
pvcName := "clawreef-team-28-shared"
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: pvcName,
Namespace: namespace,
UID: types.UID("pvc-uid"),
ResourceVersion: "1",
},
Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound},
}
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "workspace-store",
Namespace: "clawmanager-system",
},
Spec: corev1.ServiceSpec{ClusterIP: "10.96.0.44"},
}
clientset := fake.NewSimpleClientset(pvc, service)
pvcService := &PVCService{
client: &Client{
Clientset: clientset,
WorkspaceRoot: t.TempDir(),
WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
WorkspaceNFSPath: "/",
},
}
if _, err := pvcService.createPVForTeamSharedPVC(ctx, namespace, pvcName, 1, 28, 10, "manual"); err != nil {
t.Fatalf("createPVForTeamSharedPVC returned error: %v", err)
}
pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-user-1-team-28-shared", metav1.GetOptions{})
if err != nil {
t.Fatalf("expected Team shared PV: %v", err)
}
if pv.Spec.NFS == nil || pv.Spec.NFS.Server != "10.96.0.44" {
t.Fatalf("expected resolved ClusterIP NFS server, got %#v", pv.Spec.PersistentVolumeSource)
}
}
func TestCreateTeamSharedPVCCreatesWorkspaceDirectoryBeforeReturning(t *testing.T) {
ctx := context.Background()
workspaceRoot := t.TempDir()
client := &Client{
Clientset: fake.NewSimpleClientset(),
Namespace: "clawmanager",
StorageClass: "manual",
WorkspaceRoot: workspaceRoot,
WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
WorkspaceNFSPath: "/",
}
service := &PVCService{
client: client,
namespaceService: &NamespaceService{client: client},
}
if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil {
t.Fatalf("CreateTeamSharedPVC returned error: %v", err)
}
if _, err := os.Stat(filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared")); err != nil {
t.Fatalf("expected Team shared runtime workspace directory before Lite gateways start: %v", err)
}
}
func TestCreateTeamSharedPVCCreatesWritableRuntimeSubdirectories(t *testing.T) {
ctx := context.Background()
workspaceRoot := t.TempDir()
client := &Client{
Clientset: fake.NewSimpleClientset(),
Namespace: "clawmanager",
StorageClass: "manual",
WorkspaceRoot: workspaceRoot,
WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
WorkspaceNFSPath: "/",
}
service := &PVCService{
client: client,
namespaceService: &NamespaceService{client: client},
}
if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil {
t.Fatalf("CreateTeamSharedPVC returned error: %v", err)
}
for _, name := range []string{"status", "inbox", "results", "tasks"} {
dir := filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared", name)
info, err := os.Stat(dir)
if err != nil {
t.Fatalf("expected Team shared runtime subdirectory %s before gateways start: %v", name, err)
}
if !info.IsDir() {
t.Fatalf("expected %s to be a directory", dir)
}
}
}
func TestCreateTeamSharedPVCCreatesWorkspaceNFSPVBeforeReturning(t *testing.T) {
ctx := context.Background()
client := &Client{
Clientset: fake.NewSimpleClientset(),
Namespace: "clawmanager",
StorageClass: "manual",
WorkspaceRoot: t.TempDir(),
WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
WorkspaceNFSPath: "/",
}
service := &PVCService{
client: client,
namespaceService: &NamespaceService{client: client},
}
if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil {
t.Fatalf("CreateTeamSharedPVC returned error: %v", err)
}
pv, err := client.Clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-user-1-team-28-shared", metav1.GetOptions{})
if err != nil {
t.Fatalf("expected Team shared NFS PV before Lite gateways start: %v", err)
}
if pv.Spec.NFS == nil || pv.Spec.NFS.Path != "/teams/user-1/team-28-shared" {
t.Fatalf("unexpected Team shared PV source: %#v", pv.Spec.PersistentVolumeSource)
}
}
func TestCreateTeamSharedPVCRejectsExistingNonWorkspaceNFSPV(t *testing.T) {
ctx := context.Background()
namespace := "clawmanager-user-1"
pvcName := "clawreef-team-28-shared"
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: pvcName,
Namespace: namespace,
Labels: map[string]string{"team-id": "28"},
},
}
pv := &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{Name: "clawreef-pv-user-1-team-28-shared"},
Spec: corev1.PersistentVolumeSpec{
ClaimRef: &corev1.ObjectReference{Namespace: namespace, Name: pvcName},
PersistentVolumeSource: corev1.PersistentVolumeSource{
HostPath: &corev1.HostPathVolumeSource{Path: "/data/clawreef/user-1/team-28-shared"},
},
},
}
client := &Client{
Clientset: fake.NewSimpleClientset(pvc, pv),
Namespace: "clawmanager",
StorageClass: "manual",
WorkspaceRoot: t.TempDir(),
WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
WorkspaceNFSPath: "/",
}
service := &PVCService{
client: client,
namespaceService: &NamespaceService{client: client},
}
_, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual")
if err == nil || !strings.Contains(err.Error(), "workspace NFS") {
t.Fatalf("expected existing hostPath Team PV to be rejected, got %v", err)
}
}
func TestHostPathPVNodeAffinitySkipsHardTaintedNodes(t *testing.T) {
service := &PVCService{
client: &Client{
backend/internal/services/k8s/runtime_deployment_service.go
+454
View File
@@ -0,0 +1,454 @@
package k8s
import (
"context"
"encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/retry"
)
const (
runtimeAgentPort int32 = 19090
runtimeWorkspaceVolume = "workspaces"
defaultWorkspaceMount = "/workspaces"
defaultGatewayPortStart = 20000
defaultGatewayPortEnd = 20099
hermesTUIDir = "/usr/local/lib/hermes-agent/ui-tui"
)
type RuntimeDeploymentSpec struct {
Name string
Namespace string
RuntimeType string
Image string
Replicas int32
WorkspaceNFSServer string
WorkspaceNFSPath string
WorkspaceMountPath string
GatewayPortStart int
GatewayPortEnd int
AgentControlToken string
AgentReportToken string
BackendURL string
TrustedProxyCIDRs string
}
type RuntimeDeploymentPod struct {
RuntimeType string
Namespace string
DeploymentName string
PodName string
PodIP *string
NodeName *string
ImageRef string
State string
}
type RuntimeDeploymentService interface {
Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error
Scale(ctx context.Context, namespace, name string, replicas int32) error
RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error
ListPods(ctx context.Context, namespace, runtimeType string) ([]RuntimeDeploymentPod, error)
}
type runtimeDeploymentService struct {
client kubernetes.Interface
}
func NewRuntimeDeploymentService(client kubernetes.Interface) RuntimeDeploymentService {
return &runtimeDeploymentService{client: client}
}
func BuildRuntimeDeployment(spec RuntimeDeploymentSpec) *appsv1.Deployment {
labels := map[string]string{
"app": spec.Name,
"clawmanager.io/runtime-type": spec.RuntimeType,
}
replicas := spec.Replicas
workspaceMountPath := runtimeWorkspaceMountPath(spec.WorkspaceMountPath)
gatewayPortStart := runtimeGatewayPortValue(spec.GatewayPortStart, defaultGatewayPortStart)
gatewayPortEnd := runtimeGatewayPortValue(spec.GatewayPortEnd, defaultGatewayPortEnd)
env := buildRuntimeAgentEnv(spec, workspaceMountPath, gatewayPortStart, gatewayPortEnd)
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: spec.Name,
Namespace: spec.Namespace,
Labels: copyStringMap(labels),
},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: copyStringMap(labels),
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: copyStringMap(labels),
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runtime",
Image: spec.Image,
Ports: []corev1.ContainerPort{
{
Name: "agent",
ContainerPort: runtimeAgentPort,
},
},
Env: env,
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: runtimeWorkspaceVolume,
MountPath: workspaceMountPath,
},
},
},
},
Volumes: []corev1.Volume{
{
Name: runtimeWorkspaceVolume,
VolumeSource: corev1.VolumeSource{
NFS: &corev1.NFSVolumeSource{
Server: spec.WorkspaceNFSServer,
Path: spec.WorkspaceNFSPath,
},
},
},
},
},
},
},
}
}
func buildRuntimeAgentEnv(spec RuntimeDeploymentSpec, workspaceRoot string, gatewayPortStart, gatewayPortEnd int) []corev1.EnvVar {
env := []corev1.EnvVar{
{Name: "CLAWMANAGER_RUNTIME_TYPE", Value: spec.RuntimeType},
{Name: "CLAWMANAGER_BACKEND_URL", Value: runtimeBackendURL(spec)},
{Name: "CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME", Value: spec.Name},
{Name: "CLAWMANAGER_RUNTIME_IMAGE_REF", Value: spec.Image},
{Name: "CLAWMANAGER_AGENT_PORT", Value: "19090"},
{Name: "CLAWMANAGER_GATEWAY_PORT_START", Value: strconv.Itoa(gatewayPortStart)},
{Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: strconv.Itoa(gatewayPortEnd)},
{Name: "CLAWMANAGER_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken},
{Name: "CLAWMANAGER_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken},
{Name: "RUNTIME_WORKSPACE_ROOT", Value: workspaceRoot},
{Name: "RUNTIME_AGENT_LISTEN_ADDR", Value: "0.0.0.0:19090"},
{Name: "RUNTIME_AGENT_PUBLIC_PORT", Value: "19090"},
{Name: "RUNTIME_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken},
{Name: "RUNTIME_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken},
{Name: "RUNTIME_GATEWAY_PORT_START", Value: strconv.Itoa(gatewayPortStart)},
{Name: "RUNTIME_GATEWAY_PORT_END", Value: strconv.Itoa(gatewayPortEnd)},
}
if strings.EqualFold(spec.RuntimeType, "hermes") {
env = append(env, corev1.EnvVar{Name: "HERMES_TUI_DIR", Value: hermesTUIDir})
}
env = append(env,
fieldRefEnv("POD_NAME", "metadata.name"),
fieldRefEnv("POD_NAMESPACE", "metadata.namespace"),
fieldRefEnv("POD_IP", "status.podIP"),
fieldRefEnv("NODE_NAME", "spec.nodeName"),
)
if spec.TrustedProxyCIDRs != "" {
env = append(env, corev1.EnvVar{Name: "CLAWMANAGER_TRUSTED_PROXY_CIDRS", Value: spec.TrustedProxyCIDRs})
}
return env
}
func runtimeBackendURL(spec RuntimeDeploymentSpec) string {
if spec.BackendURL != "" {
return spec.BackendURL
}
namespace := spec.Namespace
if namespace == "" {
namespace = "clawmanager-system"
}
return fmt.Sprintf("http://clawmanager-gateway.%s.svc.cluster.local:9001", namespace)
}
func fieldRefEnv(name, fieldPath string) corev1.EnvVar {
return corev1.EnvVar{
Name: name,
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{FieldPath: fieldPath},
},
}
}
func runtimeWorkspaceMountPath(value string) string {
if value == "" {
return defaultWorkspaceMount
}
return value
}
func runtimeGatewayPortValue(value, defaultValue int) int {
if value == 0 {
return defaultValue
}
return value
}
func (s *runtimeDeploymentService) Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error {
if s == nil || s.client == nil {
return fmt.Errorf("k8s client not initialized")
}
desired := BuildRuntimeDeployment(spec)
deployments := s.client.AppsV1().Deployments(spec.Namespace)
existing, err := deployments.Get(ctx, spec.Name, metav1.GetOptions{})
if errors.IsNotFound(err) {
_, createErr := deployments.Create(ctx, desired, metav1.CreateOptions{})
if createErr != nil {
return fmt.Errorf("failed to create runtime deployment %s/%s: %w", spec.Namespace, spec.Name, createErr)
}
return nil
}
if err != nil {
return fmt.Errorf("failed to get runtime deployment %s/%s: %w", spec.Namespace, spec.Name, err)
}
if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) {
return fmt.Errorf("runtime deployment %s/%s selector mismatch; delete and recreate the deployment to change immutable selector", spec.Namespace, spec.Name)
}
updated := existing.DeepCopy()
if updated.Labels == nil {
updated.Labels = map[string]string{}
}
for key, value := range desired.Labels {
updated.Labels[key] = value
}
updated.Spec.Replicas = desired.Spec.Replicas
if updated.Spec.Template.Labels == nil {
updated.Spec.Template.Labels = map[string]string{}
}
for key, value := range desired.Spec.Template.Labels {
updated.Spec.Template.Labels[key] = value
}
updated.Spec.Template.Spec = desired.Spec.Template.Spec
_, err = deployments.Update(ctx, updated, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("failed to update runtime deployment %s/%s: %w", spec.Namespace, spec.Name, err)
}
return nil
}
func (s *runtimeDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error {
if s == nil || s.client == nil {
return fmt.Errorf("k8s client not initialized")
}
deployments := s.client.AppsV1().Deployments(namespace)
patch, err := json.Marshal(map[string]interface{}{
"spec": map[string]interface{}{
"replicas": replicas,
},
})
if err != nil {
return fmt.Errorf("failed to build runtime deployment scale patch %s/%s: %w", namespace, name, err)
}
if _, err := deployments.Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil {
return fmt.Errorf("failed to scale runtime deployment %s/%s: %w", namespace, name, err)
}
return nil
}
func (s *runtimeDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error {
if s == nil || s.client == nil {
return fmt.Errorf("k8s client not initialized")
}
image = strings.TrimSpace(image)
if image == "" {
return fmt.Errorf("runtime rollout image is required")
}
deployments := s.client.AppsV1().Deployments(namespace)
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
existing, err := deployments.Get(ctx, name, metav1.GetOptions{})
if err != nil {
return err
}
updated := existing.DeepCopy()
containerIndex := -1
for index, container := range updated.Spec.Template.Spec.Containers {
if container.Name == "runtime" {
containerIndex = index
break
}
}
if containerIndex < 0 {
return fmt.Errorf("runtime deployment %s/%s has no runtime container", namespace, name)
}
container := &updated.Spec.Template.Spec.Containers[containerIndex]
container.Image = image
upsertEnvVar(container, "CLAWMANAGER_RUNTIME_IMAGE_REF", image)
updated.Spec.Strategy.Type = appsv1.RollingUpdateDeploymentStrategyType
updated.Spec.Strategy.RollingUpdate = &appsv1.RollingUpdateDeployment{
MaxUnavailable: intOrStringPtr(positiveRolloutInt(maxUnavailable)),
MaxSurge: intOrStringPtr(positiveRolloutInt(maxSurge)),
}
_, err = deployments.Update(ctx, updated, metav1.UpdateOptions{})
return err
})
if err != nil {
return fmt.Errorf("failed to update runtime deployment %s/%s image: %w", namespace, name, err)
}
return nil
}
func (s *runtimeDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]RuntimeDeploymentPod, error) {
if s == nil || s.client == nil {
return nil, fmt.Errorf("k8s client not initialized")
}
namespace = strings.TrimSpace(namespace)
if namespace == "" {
return nil, fmt.Errorf("runtime namespace is required")
}
runtimeType = strings.ToLower(strings.TrimSpace(runtimeType))
listOptions := metav1.ListOptions{}
if runtimeType != "" {
listOptions.LabelSelector = labels.Set{"clawmanager.io/runtime-type": runtimeType}.String()
}
deploymentList, err := s.client.AppsV1().Deployments(namespace).List(ctx, listOptions)
if err != nil {
return nil, fmt.Errorf("failed to list runtime deployments in %s: %w", namespace, err)
}
var pods []RuntimeDeploymentPod
for _, deployment := range deploymentList.Items {
deploymentRuntimeType := strings.ToLower(strings.TrimSpace(deployment.Labels["clawmanager.io/runtime-type"]))
if deploymentRuntimeType == "" {
continue
}
if runtimeType != "" && deploymentRuntimeType != runtimeType {
continue
}
if deployment.Spec.Selector == nil {
return nil, fmt.Errorf("runtime deployment %s/%s has no selector", deployment.Namespace, deployment.Name)
}
selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
if err != nil {
return nil, fmt.Errorf("runtime deployment %s/%s has invalid selector: %w", deployment.Namespace, deployment.Name, err)
}
podList, err := s.client.CoreV1().Pods(deployment.Namespace).List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
if err != nil {
return nil, fmt.Errorf("failed to list runtime deployment pods %s/%s: %w", deployment.Namespace, deployment.Name, err)
}
deploymentImage := runtimeContainerImage(deployment.Spec.Template.Spec.Containers)
for _, pod := range podList.Items {
image := runtimeContainerImage(pod.Spec.Containers)
if image == "" {
image = deploymentImage
}
pods = append(pods, RuntimeDeploymentPod{
RuntimeType: deploymentRuntimeType,
Namespace: pod.Namespace,
DeploymentName: deployment.Name,
PodName: pod.Name,
PodIP: stringPtrIfNotEmpty(pod.Status.PodIP),
NodeName: stringPtrIfNotEmpty(pod.Spec.NodeName),
ImageRef: image,
State: runtimeK8sPodState(pod),
})
}
}
return pods, nil
}
func runtimeContainerImage(containers []corev1.Container) string {
for _, container := range containers {
if container.Name == "runtime" {
return strings.TrimSpace(container.Image)
}
}
if len(containers) == 1 {
return strings.TrimSpace(containers[0].Image)
}
return ""
}
func runtimeK8sPodState(pod corev1.Pod) string {
if pod.DeletionTimestamp != nil {
return "deleted"
}
switch pod.Status.Phase {
case corev1.PodPending:
return "pending"
case corev1.PodRunning:
if runtimeK8sPodReady(pod) {
return "ready"
}
return "pending"
case corev1.PodSucceeded, corev1.PodFailed:
return "unhealthy"
default:
return "pending"
}
}
func runtimeK8sPodReady(pod corev1.Pod) bool {
for _, condition := range pod.Status.Conditions {
if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
return true
}
}
return false
}
func stringPtrIfNotEmpty(value string) *string {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
return &value
}
func upsertEnvVar(container *corev1.Container, name, value string) {
for index := range container.Env {
if container.Env[index].Name == name {
container.Env[index].Value = value
container.Env[index].ValueFrom = nil
return
}
}
container.Env = append(container.Env, corev1.EnvVar{Name: name, Value: value})
}
func positiveRolloutInt(value int) int {
if value <= 0 {
return 1
}
return value
}
func intOrStringPtr(value int) *intstr.IntOrString {
result := intstr.FromInt(value)
return &result
}
backend/internal/services/k8s/runtime_deployment_service_test.go
+481
View File
@@ -0,0 +1,481 @@
package k8s
import (
"context"
"fmt"
"testing"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes/fake"
k8stesting "k8s.io/client-go/testing"
)
func TestBuildRuntimeDeployment(t *testing.T) {
deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
Name: "runtime-openclaw",
Namespace: "runtime-system",
RuntimeType: "openclaw",
Image: "registry/openclaw:latest",
Replicas: 2,
WorkspaceNFSServer: "10.0.0.9",
WorkspaceNFSPath: "/exports/workspaces",
AgentControlToken: "control-secret",
AgentReportToken: "report-secret",
})
if deployment.Name != "runtime-openclaw" || deployment.Namespace != "runtime-system" {
t.Fatalf("unexpected deployment identity: %s/%s", deployment.Namespace, deployment.Name)
}
if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 2 {
t.Fatalf("expected replicas 2, got %#v", deployment.Spec.Replicas)
}
if got := deployment.Labels["clawmanager.io/runtime-type"]; got != "openclaw" {
t.Fatalf("expected runtime-type label openclaw, got %q", got)
}
if len(deployment.Spec.Template.Spec.Containers) != 1 {
t.Fatalf("expected one container, got %d", len(deployment.Spec.Template.Spec.Containers))
}
container := deployment.Spec.Template.Spec.Containers[0]
if container.Name != "runtime" || container.Image != "registry/openclaw:latest" {
t.Fatalf("unexpected runtime container: %#v", container)
}
requireContainerPort(t, container, "agent", 19090)
requireVolumeMount(t, container, "workspaces", "/workspaces")
requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "openclaw")
requireEnv(t, container, "CLAWMANAGER_AGENT_PORT", "19090")
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "20000")
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "20099")
requireEnv(t, container, "CLAWMANAGER_AGENT_CONTROL_TOKEN", "control-secret")
requireEnv(t, container, "CLAWMANAGER_AGENT_REPORT_TOKEN", "report-secret")
if got := container.Resources.Requests[corev1.ResourceCPU]; got.Cmp(resource.MustParse("500m")) != 0 {
t.Fatalf("expected CPU request 500m, got %s", got.String())
}
if got := container.Resources.Requests[corev1.ResourceMemory]; got.Cmp(resource.MustParse("1Gi")) != 0 {
t.Fatalf("expected memory request 1Gi, got %s", got.String())
}
if len(deployment.Spec.Template.Spec.Volumes) != 1 {
t.Fatalf("expected one volume, got %#v", deployment.Spec.Template.Spec.Volumes)
}
volume := deployment.Spec.Template.Spec.Volumes[0]
if volume.Name != "workspaces" || volume.NFS == nil {
t.Fatalf("expected workspaces NFS volume, got %#v", volume)
}
if volume.NFS.Server != "10.0.0.9" || volume.NFS.Path != "/exports/workspaces" {
t.Fatalf("unexpected NFS source: %#v", volume.NFS)
}
}
func TestBuildRuntimeDeploymentKeepsCapacityLimitOutOfRuntimeEnv(t *testing.T) {
deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
Name: "runtime-openclaw",
Namespace: "runtime-system",
RuntimeType: "openclaw",
Image: "registry/openclaw:latest",
Replicas: 2,
WorkspaceNFSServer: "10.0.0.9",
WorkspaceNFSPath: "/exports/workspaces",
WorkspaceMountPath: "/runtime-workspaces",
GatewayPortStart: 21000,
GatewayPortEnd: 21049,
})
container := deployment.Spec.Template.Spec.Containers[0]
requireVolumeMount(t, container, "workspaces", "/runtime-workspaces")
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "21000")
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "21049")
requireEnvAbsent(t, container, "CLAWMANAGER_MAX_GATEWAYS_PER_POD")
requireEnvAbsent(t, container, "RUNTIME_MAX_GATEWAYS_PER_POD")
}
func TestBuildRuntimeDeploymentInjectsAgentV2Environment(t *testing.T) {
deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
Name: "runtime-hermes",
Namespace: "runtime-system",
RuntimeType: "hermes",
Image: "registry/hermes:v2",
Replicas: 1,
WorkspaceNFSServer: "10.0.0.9",
WorkspaceNFSPath: "/exports/workspaces",
WorkspaceMountPath: "/workspaces",
GatewayPortStart: 21000,
GatewayPortEnd: 21099,
AgentControlToken: "control-secret",
AgentReportToken: "report-secret",
BackendURL: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001",
TrustedProxyCIDRs: "10.42.0.0/16,10.43.0.0/16",
})
container := deployment.Spec.Template.Spec.Containers[0]
requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "hermes")
requireEnv(t, container, "CLAWMANAGER_BACKEND_URL", "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001")
requireEnv(t, container, "CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME", "runtime-hermes")
requireEnv(t, container, "CLAWMANAGER_RUNTIME_IMAGE_REF", "registry/hermes:v2")
requireEnv(t, container, "RUNTIME_WORKSPACE_ROOT", "/workspaces")
requireEnv(t, container, "RUNTIME_AGENT_LISTEN_ADDR", "0.0.0.0:19090")
requireEnv(t, container, "RUNTIME_AGENT_PUBLIC_PORT", "19090")
requireEnv(t, container, "RUNTIME_AGENT_CONTROL_TOKEN", "control-secret")
requireEnv(t, container, "RUNTIME_AGENT_REPORT_TOKEN", "report-secret")
requireEnv(t, container, "RUNTIME_GATEWAY_PORT_START", "21000")
requireEnv(t, container, "RUNTIME_GATEWAY_PORT_END", "21099")
requireEnv(t, container, "HERMES_TUI_DIR", "/usr/local/lib/hermes-agent/ui-tui")
requireEnv(t, container, "CLAWMANAGER_TRUSTED_PROXY_CIDRS", "10.42.0.0/16,10.43.0.0/16")
requireEnvFieldRef(t, container, "POD_NAME", "metadata.name")
requireEnvFieldRef(t, container, "POD_NAMESPACE", "metadata.namespace")
requireEnvFieldRef(t, container, "POD_IP", "status.podIP")
requireEnvFieldRef(t, container, "NODE_NAME", "spec.nodeName")
}
func TestRuntimeDeploymentServiceEnsureCreatesAndUpdates(t *testing.T) {
client := fake.NewSimpleClientset()
service := NewRuntimeDeploymentService(client)
spec := RuntimeDeploymentSpec{
Name: "runtime-hermes",
Namespace: "runtime-system",
RuntimeType: "hermes",
Image: "registry/hermes:v1",
Replicas: 1,
WorkspaceNFSServer: "nfs.local",
WorkspaceNFSPath: "/exports",
}
if err := service.Ensure(context.Background(), spec); err != nil {
t.Fatalf("Ensure create returned error: %v", err)
}
created, err := client.AppsV1().Deployments(spec.Namespace).Get(context.Background(), spec.Name, metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get created deployment: %v", err)
}
if created.Spec.Replicas == nil || *created.Spec.Replicas != 1 {
t.Fatalf("expected created replicas 1, got %#v", created.Spec.Replicas)
}
spec.Image = "registry/hermes:v2"
spec.Replicas = 3
if err := service.Ensure(context.Background(), spec); err != nil {
t.Fatalf("Ensure update returned error: %v", err)
}
updated, err := client.AppsV1().Deployments(spec.Namespace).Get(context.Background(), spec.Name, metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get updated deployment: %v", err)
}
if updated.Spec.Template.Spec.Containers[0].Image != "registry/hermes:v2" {
t.Fatalf("expected updated image, got %q", updated.Spec.Template.Spec.Containers[0].Image)
}
if updated.Spec.Replicas == nil || *updated.Spec.Replicas != 3 {
t.Fatalf("expected updated replicas 3, got %#v", updated.Spec.Replicas)
}
}
func TestRuntimeDeploymentServiceEnsurePreservesUnownedMetadata(t *testing.T) {
existing := BuildRuntimeDeployment(RuntimeDeploymentSpec{
Name: "runtime-openclaw",
Namespace: "runtime-system",
RuntimeType: "openclaw",
Image: "registry/openclaw:v1",
Replicas: 1,
WorkspaceNFSServer: "nfs.local",
WorkspaceNFSPath: "/exports",
})
existing.Labels["admission.example.com/injected"] = "keep"
existing.Annotations = map[string]string{"admission.example.com/sidecar": "enabled"}
existing.Finalizers = []string{"protect.example.com/runtime"}
existing.Spec.Template.Labels["admission.example.com/template-label"] = "keep"
existing.Spec.Template.Annotations = map[string]string{"admission.example.com/template-annotation": "enabled"}
client := fake.NewSimpleClientset(existing)
service := NewRuntimeDeploymentService(client)
if err := service.Ensure(context.Background(), RuntimeDeploymentSpec{
Name: "runtime-openclaw",
Namespace: "runtime-system",
RuntimeType: "openclaw",
Image: "registry/openclaw:v2",
Replicas: 3,
WorkspaceNFSServer: "nfs.local",
WorkspaceNFSPath: "/exports",
}); err != nil {
t.Fatalf("Ensure update returned error: %v", err)
}
updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-openclaw", metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get updated deployment: %v", err)
}
if updated.Labels["admission.example.com/injected"] != "keep" {
t.Fatalf("expected unrelated label to survive, got %#v", updated.Labels)
}
if updated.Annotations["admission.example.com/sidecar"] != "enabled" {
t.Fatalf("expected annotation to survive, got %#v", updated.Annotations)
}
if len(updated.Finalizers) != 1 || updated.Finalizers[0] != "protect.example.com/runtime" {
t.Fatalf("expected finalizer to survive, got %#v", updated.Finalizers)
}
if updated.Labels["clawmanager.io/runtime-type"] != "openclaw" {
t.Fatalf("expected owned runtime label to be present, got %#v", updated.Labels)
}
if updated.Spec.Template.Labels["admission.example.com/template-label"] != "keep" {
t.Fatalf("expected unrelated template label to survive, got %#v", updated.Spec.Template.Labels)
}
if updated.Spec.Template.Annotations["admission.example.com/template-annotation"] != "enabled" {
t.Fatalf("expected template annotation to survive, got %#v", updated.Spec.Template.Annotations)
}
if updated.Spec.Template.Labels["clawmanager.io/runtime-type"] != "openclaw" {
t.Fatalf("expected owned template runtime label to be present, got %#v", updated.Spec.Template.Labels)
}
if updated.Spec.Template.Spec.Containers[0].Image != "registry/openclaw:v2" {
t.Fatalf("expected image update while preserving template metadata, got %q", updated.Spec.Template.Spec.Containers[0].Image)
}
}
func TestRuntimeDeploymentServiceEnsureRejectsSelectorChange(t *testing.T) {
existing := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "runtime-openclaw", Namespace: "runtime-system"},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "different"}},
},
}
client := fake.NewSimpleClientset(existing)
service := NewRuntimeDeploymentService(client)
err := service.Ensure(context.Background(), RuntimeDeploymentSpec{
Name: "runtime-openclaw",
Namespace: "runtime-system",
RuntimeType: "openclaw",
Image: "registry/openclaw:latest",
Replicas: 1,
})
if err == nil {
t.Fatalf("expected selector mismatch error")
}
}
func TestRuntimeDeploymentServiceScale(t *testing.T) {
deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
Name: "runtime-openclaw",
Namespace: "runtime-system",
RuntimeType: "openclaw",
Image: "registry/openclaw:latest",
Replicas: 1,
WorkspaceNFSServer: "nfs.local",
WorkspaceNFSPath: "/exports",
})
deployment.Spec.Template.Annotations = map[string]string{"admission.example.com/template": "keep"}
client := fake.NewSimpleClientset(deployment)
service := NewRuntimeDeploymentService(client)
if err := service.Scale(context.Background(), "runtime-system", "runtime-openclaw", 4); err != nil {
t.Fatalf("Scale returned error: %v", err)
}
scaled, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-openclaw", metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get scaled deployment: %v", err)
}
if scaled.Spec.Replicas == nil || *scaled.Spec.Replicas != 4 {
t.Fatalf("expected replicas 4, got %#v", scaled.Spec.Replicas)
}
if scaled.Spec.Template.Spec.Containers[0].Image != "registry/openclaw:latest" {
t.Fatalf("expected Scale to preserve image, got %q", scaled.Spec.Template.Spec.Containers[0].Image)
}
if scaled.Spec.Template.Annotations["admission.example.com/template"] != "keep" {
t.Fatalf("expected Scale to preserve template metadata, got %#v", scaled.Spec.Template.Annotations)
}
}
func TestRuntimeDeploymentServiceRolloutImage(t *testing.T) {
deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
Name: "runtime-hermes",
Namespace: "runtime-system",
RuntimeType: "hermes",
Image: "registry/hermes:v1",
Replicas: 3,
WorkspaceNFSServer: "nfs.local",
WorkspaceNFSPath: "/exports",
})
client := fake.NewSimpleClientset(deployment)
service := NewRuntimeDeploymentService(client)
if err := service.RolloutImage(context.Background(), "runtime-system", "runtime-hermes", "registry/hermes:v2", 1, 2); err != nil {
t.Fatalf("RolloutImage returned error: %v", err)
}
updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-hermes", metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get updated deployment: %v", err)
}
container := updated.Spec.Template.Spec.Containers[0]
if container.Image != "registry/hermes:v2" {
t.Fatalf("expected updated image, got %q", container.Image)
}
requireEnv(t, container, "CLAWMANAGER_RUNTIME_IMAGE_REF", "registry/hermes:v2")
if updated.Spec.Strategy.Type != appsv1.RollingUpdateDeploymentStrategyType {
t.Fatalf("expected RollingUpdate strategy, got %q", updated.Spec.Strategy.Type)
}
if updated.Spec.Strategy.RollingUpdate == nil {
t.Fatal("expected rolling update strategy")
}
if got := updated.Spec.Strategy.RollingUpdate.MaxUnavailable.IntValue(); got != 1 {
t.Fatalf("maxUnavailable = %d, want 1", got)
}
if got := updated.Spec.Strategy.RollingUpdate.MaxSurge.IntValue(); got != 2 {
t.Fatalf("maxSurge = %d, want 2", got)
}
}
func TestRuntimeDeploymentServiceRolloutImageRetriesConflict(t *testing.T) {
deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
Name: "runtime-hermes",
Namespace: "runtime-system",
RuntimeType: "hermes",
Image: "registry/hermes:v1",
Replicas: 1,
WorkspaceNFSServer: "nfs.local",
WorkspaceNFSPath: "/exports",
})
client := fake.NewSimpleClientset(deployment)
updateAttempts := 0
client.Fake.PrependReactor("update", "deployments", func(action k8stesting.Action) (bool, runtime.Object, error) {
updateAttempts++
if updateAttempts == 1 {
return true, nil, apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "deployments"}, "runtime-hermes", fmt.Errorf("stale resource version"))
}
return false, nil, nil
})
service := NewRuntimeDeploymentService(client)
if err := service.RolloutImage(context.Background(), "runtime-system", "runtime-hermes", "registry/hermes:v2", 1, 1); err != nil {
t.Fatalf("RolloutImage returned error: %v", err)
}
if updateAttempts < 2 {
t.Fatalf("update attempts = %d, want retry after conflict", updateAttempts)
}
updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-hermes", metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get updated deployment: %v", err)
}
if got := updated.Spec.Template.Spec.Containers[0].Image; got != "registry/hermes:v2" {
t.Fatalf("image = %q, want registry/hermes:v2", got)
}
}
func TestRuntimeDeploymentServiceListPodsReturnsRuntimeDeploymentPods(t *testing.T) {
deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
Name: "openclaw-runtime",
Namespace: "runtime-system",
RuntimeType: "openclaw",
Image: "registry/openclaw-lite:final2",
Replicas: 1,
WorkspaceNFSServer: "nfs.local",
WorkspaceNFSPath: "/exports",
})
podIP := "10.42.0.12"
nodeName := "node-a"
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "openclaw-runtime-abc",
Namespace: "runtime-system",
Labels: map[string]string{
"app": "openclaw-runtime",
"clawmanager.io/runtime-type": "openclaw",
},
},
Spec: corev1.PodSpec{
NodeName: nodeName,
Containers: []corev1.Container{{
Name: "runtime",
Image: "registry/openclaw-lite:final2",
}},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
PodIP: podIP,
Conditions: []corev1.PodCondition{{
Type: corev1.PodReady,
Status: corev1.ConditionTrue,
}},
},
}
client := fake.NewSimpleClientset(deployment, pod)
service := NewRuntimeDeploymentService(client)
pods, err := service.ListPods(context.Background(), "runtime-system", "openclaw")
if err != nil {
t.Fatalf("ListPods returned error: %v", err)
}
if len(pods) != 1 {
t.Fatalf("pods length = %d, want 1: %#v", len(pods), pods)
}
got := pods[0]
if got.RuntimeType != "openclaw" || got.Namespace != "runtime-system" || got.DeploymentName != "openclaw-runtime" || got.PodName != "openclaw-runtime-abc" {
t.Fatalf("runtime pod identity = %+v, want openclaw runtime pod", got)
}
if got.ImageRef != "registry/openclaw-lite:final2" {
t.Fatalf("image = %q, want registry/openclaw-lite:final2", got.ImageRef)
}
if got.State != "ready" {
t.Fatalf("state = %q, want ready", got.State)
}
if got.PodIP == nil || *got.PodIP != podIP || got.NodeName == nil || *got.NodeName != nodeName {
t.Fatalf("pod network fields = ip:%v node:%v, want %s/%s", got.PodIP, got.NodeName, podIP, nodeName)
}
}
func requireContainerPort(t *testing.T, container corev1.Container, name string, port int32) {
t.Helper()
for _, got := range container.Ports {
if got.Name == name && got.ContainerPort == port {
return
}
}
t.Fatalf("expected container port %s=%d, got %#v", name, port, container.Ports)
}
func requireVolumeMount(t *testing.T, container corev1.Container, name, mountPath string) {
t.Helper()
for _, got := range container.VolumeMounts {
if got.Name == name && got.MountPath == mountPath {
return
}
}
t.Fatalf("expected volume mount %s at %s, got %#v", name, mountPath, container.VolumeMounts)
}
func requireEnv(t *testing.T, container corev1.Container, name, value string) {
t.Helper()
for _, got := range container.Env {
if got.Name == name && got.Value == value {
return
}
}
t.Fatalf("expected env %s=%q, got %#v", name, value, container.Env)
}
func requireEnvAbsent(t *testing.T, container corev1.Container, name string) {
t.Helper()
for _, got := range container.Env {
if got.Name == name {
t.Fatalf("expected env %s to be absent, got %#v", name, container.Env)
}
}
}
func requireEnvFieldRef(t *testing.T, container corev1.Container, name, fieldPath string) {
t.Helper()
for _, got := range container.Env {
if got.Name != name || got.ValueFrom == nil || got.ValueFrom.FieldRef == nil {
continue
}
if got.ValueFrom.FieldRef.FieldPath == fieldPath {
return
}
}
t.Fatalf("expected env %s from fieldRef %q, got %#v", name, fieldPath, container.Env)
}
backend/internal/services/openclaw_transfer_service.go
+158 -16
View File
@@ -7,7 +7,10 @@ import (
"fmt"
"io"
"strings"
"sync"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services/k8s"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes/scheme"
@@ -39,9 +42,29 @@ type OpenClawTransferService interface {
}
type openClawTransferService struct {
podService *k8s.PodService
podService *k8s.PodService
instanceRepo repository.InstanceRepository
bindingRepo repository.InstanceRuntimeBindingRepository
runtimePodRepo repository.RuntimePodRepository
}
type openClawTransferRuntimeRepositories struct {
instanceRepo repository.InstanceRepository
bindingRepo repository.InstanceRuntimeBindingRepository
runtimePodRepo repository.RuntimePodRepository
}
type transferExecTarget struct {
namespace string
podName string
container string
}
var (
openClawTransferReposMu sync.RWMutex
openClawTransferRepos openClawTransferRuntimeRepositories
)
type workspaceTransferSpec struct {
dirName string
baseDirExpr string
@@ -51,11 +74,27 @@ type workspaceTransferSpec struct {
}
func NewOpenClawTransferService() OpenClawTransferService {
openClawTransferReposMu.RLock()
repos := openClawTransferRepos
openClawTransferReposMu.RUnlock()
return &openClawTransferService{
podService: k8s.NewPodService(),
podService: k8s.NewPodService(),
instanceRepo: repos.instanceRepo,
bindingRepo: repos.bindingRepo,
runtimePodRepo: repos.runtimePodRepo,
}
}
func SetOpenClawTransferRuntimeRepositories(instanceRepo repository.InstanceRepository, bindingRepo repository.InstanceRuntimeBindingRepository, runtimePodRepo repository.RuntimePodRepository) {
openClawTransferReposMu.Lock()
openClawTransferRepos = openClawTransferRuntimeRepositories{
instanceRepo: instanceRepo,
bindingRepo: bindingRepo,
runtimePodRepo: runtimePodRepo,
}
openClawTransferReposMu.Unlock()
}
// buildBaseDirExpr returns a POSIX shell expression that resolves the
// OpenClaw persistent directory inside the desktop container. It honors the
// CLAWMANAGER_AGENT_PERSISTENT_DIR env var (injected by ClawManager at pod
@@ -150,21 +189,26 @@ func buildWorkspaceImportCommand(spec workspaceTransferSpec) []string {
}
func (s *openClawTransferService) Export(ctx context.Context, userID, instanceID int) ([]byte, error) {
return s.exportWorkspace(ctx, userID, instanceID, openClawWorkspaceSpec(), buildExportCommand())
return s.exportWorkspace(ctx, userID, instanceID, openClawWorkspaceSpec())
}
func (s *openClawTransferService) ExportHermes(ctx context.Context, userID, instanceID int) ([]byte, error) {
return s.exportWorkspace(ctx, userID, instanceID, hermesWorkspaceSpec(), buildHermesExportCommand())
return s.exportWorkspace(ctx, userID, instanceID, hermesWorkspaceSpec())
}
func (s *openClawTransferService) exportWorkspace(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec, command []string) ([]byte, error) {
func (s *openClawTransferService) exportWorkspace(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec) ([]byte, error) {
resolvedSpec, err := s.workspaceSpecForInstance(ctx, userID, instanceID, spec)
if err != nil {
return nil, err
}
command := buildWorkspaceExportCommand(resolvedSpec)
var stdout bytes.Buffer
var stderr bytes.Buffer
if err := s.exec(ctx, userID, instanceID, command, nil, &stdout, &stderr); err != nil {
if isExportEmptyWorkspaceError(err) {
return nil, spec.missingErr
return nil, resolvedSpec.missingErr
}
return nil, formatExecError("export "+spec.actionLabel, err, stderr.String())
return nil, formatExecError("export "+resolvedSpec.actionLabel, err, stderr.String())
}
return stdout.Bytes(), nil
@@ -187,40 +231,76 @@ func isExportEmptyWorkspaceError(err error) bool {
}
func (s *openClawTransferService) Import(ctx context.Context, userID, instanceID int, archive io.Reader) error {
return s.importWorkspace(ctx, userID, instanceID, archive, openClawWorkspaceSpec(), buildImportCommand())
return s.importWorkspace(ctx, userID, instanceID, archive, openClawWorkspaceSpec())
}
func (s *openClawTransferService) ImportHermes(ctx context.Context, userID, instanceID int, archive io.Reader) error {
return s.importWorkspace(ctx, userID, instanceID, archive, hermesWorkspaceSpec(), buildHermesImportCommand())
return s.importWorkspace(ctx, userID, instanceID, archive, hermesWorkspaceSpec())
}
func (s *openClawTransferService) importWorkspace(ctx context.Context, userID, instanceID int, archive io.Reader, spec workspaceTransferSpec, command []string) error {
func (s *openClawTransferService) importWorkspace(ctx context.Context, userID, instanceID int, archive io.Reader, spec workspaceTransferSpec) error {
resolvedSpec, err := s.workspaceSpecForInstance(ctx, userID, instanceID, spec)
if err != nil {
return err
}
command := buildWorkspaceImportCommand(resolvedSpec)
var stderr bytes.Buffer
if err := s.exec(ctx, userID, instanceID, command, archive, nil, &stderr); err != nil {
return formatExecError("import "+spec.actionLabel, err, stderr.String())
return formatExecError("import "+resolvedSpec.actionLabel, err, stderr.String())
}
return nil
}
func (s *openClawTransferService) workspaceSpecForInstance(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec) (workspaceTransferSpec, error) {
instance, err := s.runtimeManagedInstance(ctx, userID, instanceID)
if err != nil || instance == nil {
return spec, err
}
baseDir := ""
if instance.WorkspacePath != nil {
baseDir = strings.TrimSpace(*instance.WorkspacePath)
}
if baseDir == "" {
runtimeType, ok := NormalizeV2RuntimeType(instance.Type)
if !ok {
return spec, nil
}
baseDir = RuntimeWorkspacePath(runtimeType, instance.UserID, instance.ID)
}
spec.baseDirExpr = baseDir
return spec, nil
}
func (s *openClawTransferService) exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error {
if s.podService == nil || s.podService.GetClient() == nil || s.podService.GetClient().Clientset == nil {
return fmt.Errorf("k8s client not initialized")
}
pod, err := s.podService.GetPod(ctx, userID, instanceID)
target, err := s.runtimeExecTarget(ctx, userID, instanceID)
if err != nil {
return fmt.Errorf("failed to get pod: %w", err)
return err
}
if target == nil {
pod, podErr := s.podService.GetPod(ctx, userID, instanceID)
if podErr != nil {
return fmt.Errorf("failed to get pod: %w", podErr)
}
target = &transferExecTarget{
namespace: pod.Namespace,
podName: pod.Name,
container: "desktop",
}
}
req := s.podService.GetClient().Clientset.CoreV1().RESTClient().Post().
Resource("pods").
Name(pod.Name).
Namespace(pod.Namespace).
Name(target.podName).
Namespace(target.namespace).
SubResource("exec")
req.VersionedParams(&corev1.PodExecOptions{
Container: "desktop",
Container: target.container,
Command: command,
Stdin: stdin != nil,
Stdout: stdout != nil,
@@ -241,6 +321,68 @@ func (s *openClawTransferService) exec(ctx context.Context, userID, instanceID i
})
}
func (s *openClawTransferService) runtimeManagedInstance(ctx context.Context, userID, instanceID int) (*models.Instance, error) {
if s == nil || s.instanceRepo == nil {
return nil, nil
}
instance, err := s.instanceRepo.GetByID(instanceID)
if err != nil {
return nil, fmt.Errorf("failed to get instance: %w", err)
}
if instance == nil {
return nil, nil
}
if instance.UserID != userID {
return nil, fmt.Errorf("instance does not belong to user")
}
if !isLiteRuntimeInstance(instance) {
return nil, nil
}
return instance, nil
}
func (s *openClawTransferService) runtimeExecTarget(ctx context.Context, userID, instanceID int) (*transferExecTarget, error) {
instance, err := s.runtimeManagedInstance(ctx, userID, instanceID)
if err != nil || instance == nil {
return nil, err
}
if s.bindingRepo == nil || s.runtimePodRepo == nil {
return nil, fmt.Errorf("runtime repositories are not configured for lite workspace transfer")
}
binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID)
if err != nil {
return nil, fmt.Errorf("failed to get runtime binding: %w", err)
}
if binding == nil {
return nil, fmt.Errorf("runtime binding not found for instance %d", instanceID)
}
if binding.Generation != instance.RuntimeGeneration {
return nil, fmt.Errorf("runtime binding generation does not match instance")
}
runtimePod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
if err != nil {
return nil, fmt.Errorf("failed to get runtime pod: %w", err)
}
if runtimePod == nil {
return nil, fmt.Errorf("runtime pod not found for instance %d", instanceID)
}
return &transferExecTarget{
namespace: runtimePod.Namespace,
podName: runtimePod.PodName,
container: "runtime",
}, nil
}
func isLiteRuntimeInstance(instance *models.Instance) bool {
if instance == nil {
return false
}
if strings.EqualFold(strings.TrimSpace(instance.InstanceMode), InstanceModeLite) {
return true
}
return strings.EqualFold(strings.TrimSpace(instance.RuntimeType), RuntimeBackendGateway)
}
func shellQuote(value string) string {
return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
}
backend/internal/services/openclaw_transfer_service_test.go
+44
View File
@@ -1,8 +1,11 @@
package services
import (
"context"
"strings"
"testing"
"clawreef/internal/models"
)
func TestBuildBaseDirExpr_UsesEnvVarWithFallback(t *testing.T) {
@@ -110,6 +113,47 @@ func TestBuildHermesImportCommand_PreservesMountedHermesDirectory(t *testing.T)
}
}
func TestWorkspaceSpecForLiteUsesRuntimeWorkspacePath(t *testing.T) {
instanceRepo := newFakeRuntimeInstanceRepo()
instanceRepo.byID[123] = &models.Instance{
ID: 123,
UserID: 45,
Type: RuntimeTypeOpenClaw,
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
RuntimeGeneration: 2,
}
service := &openClawTransferService{instanceRepo: instanceRepo}
spec, err := service.workspaceSpecForInstance(context.Background(), 45, 123, openClawWorkspaceSpec())
if err != nil {
t.Fatalf("workspaceSpecForInstance returned error: %v", err)
}
if got, want := spec.baseDirExpr, RuntimeWorkspacePath(RuntimeTypeOpenClaw, 45, 123); got != want {
t.Fatalf("base dir = %q, want %q", got, want)
}
}
func TestWorkspaceSpecForProKeepsDesktopConfigPath(t *testing.T) {
instanceRepo := newFakeRuntimeInstanceRepo()
instanceRepo.byID[124] = &models.Instance{
ID: 124,
UserID: 45,
Type: RuntimeTypeOpenClaw,
RuntimeType: RuntimeBackendDesktop,
InstanceMode: InstanceModePro,
}
service := &openClawTransferService{instanceRepo: instanceRepo}
spec, err := service.workspaceSpecForInstance(context.Background(), 45, 124, openClawWorkspaceSpec())
if err != nil {
t.Fatalf("workspaceSpecForInstance returned error: %v", err)
}
if got, want := spec.baseDirExpr, buildBaseDirExpr(); got != want {
t.Fatalf("base dir = %q, want %q", got, want)
}
}
func TestShellQuote_HandlesSingleQuotes(t *testing.T) {
cases := []struct {
in, want string
backend/internal/services/redis_client.go
+17
View File
@@ -0,0 +1,17 @@
package services
import (
"context"
"time"
)
type PlatformRedisClient interface {
SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error)
Del(ctx context.Context, key string) error
XAdd(ctx context.Context, key string, fields map[string]string) (string, error)
XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error)
}
func NewPlatformRedisClient(rawURL string) (PlatformRedisClient, error) {
return newRedisBus(rawURL)
}
backend/internal/services/risk_hit_service.go
+19 -2
View File
@@ -10,10 +10,17 @@ import (
// RiskHitService defines operations for persisted risk hits.
type RiskHitService interface {
RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, action string, hits []RiskMatch) error
RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, attribution *RiskHitAttribution, action string, hits []RiskMatch) error
ListHitsByTraceID(traceID string) ([]models.RiskHit, error)
}
type RiskHitAttribution struct {
InstanceMode *string
RuntimeType *string
GatewayID *string
RuntimePodID *int64
}
type riskHitService struct {
repo repository.RiskHitRepository
}
@@ -23,7 +30,7 @@ func NewRiskHitService(repo repository.RiskHitRepository) RiskHitService {
return &riskHitService{repo: repo}
}
func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, action string, hits []RiskMatch) error {
func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, attribution *RiskHitAttribution, action string, hits []RiskMatch) error {
traceID = strings.TrimSpace(traceID)
if traceID == "" || len(hits) == 0 {
return nil
@@ -39,6 +46,10 @@ func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string
RequestID: requestID,
UserID: userID,
InstanceID: instanceID,
InstanceMode: nil,
RuntimeType: nil,
GatewayID: nil,
RuntimePodID: nil,
InvocationID: invocationID,
RuleID: strings.TrimSpace(hit.RuleID),
RuleName: strings.TrimSpace(hit.RuleName),
@@ -46,6 +57,12 @@ func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string
Action: action,
MatchSummary: strings.TrimSpace(hit.MatchSummary),
}
if attribution != nil {
record.InstanceMode = attribution.InstanceMode
record.RuntimeType = attribution.RuntimeType
record.GatewayID = attribution.GatewayID
record.RuntimePodID = attribution.RuntimePodID
}
if record.RuleID == "" || record.RuleName == "" || record.Severity == "" || record.MatchSummary == "" {
return fmt.Errorf("risk hit record is incomplete")
}
backend/internal/services/runtime_agent_client.go
+130
View File
@@ -0,0 +1,130 @@
package services
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
type RuntimeAgentClient interface {
Health(ctx context.Context, endpoint string) error
CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error)
DeleteGateway(ctx context.Context, endpoint, gatewayID string) error
Drain(ctx context.Context, endpoint string) error
}
type RuntimeAgentPortRange struct {
Start int `json:"start"`
End int `json:"end"`
}
type RuntimeAgentCreateGatewayRequest struct {
InstanceID int `json:"instance_id"`
UserID int `json:"user_id"`
AgentType string `json:"agent_type"`
WorkspacePath string `json:"workspace_path"`
PortRange RuntimeAgentPortRange `json:"port_range"`
UID int `json:"uid"`
GID int `json:"gid"`
CPUCores float64 `json:"cpu_cores"`
MemoryMB int `json:"memory_mb"`
DiskQuotaMB int `json:"disk_quota_mb"`
Generation int `json:"generation"`
Environment map[string]string `json:"environment,omitempty"`
}
type RuntimeAgentCreateGatewayResponse struct {
GatewayID string `json:"gateway_id"`
Port int `json:"port"`
PID *int `json:"pid,omitempty"`
Status string `json:"status"`
}
type runtimeAgentHTTPClient struct {
controlToken string
httpClient *http.Client
}
func NewRuntimeAgentClient(controlToken string) RuntimeAgentClient {
return NewRuntimeAgentClientWithHTTPClient(controlToken, nil)
}
func NewRuntimeAgentClientWithHTTPClient(controlToken string, httpClient *http.Client) RuntimeAgentClient {
if httpClient == nil {
httpClient = &http.Client{
Timeout: 30 * time.Second,
}
}
clientCopy := *httpClient
clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
return &runtimeAgentHTTPClient{
controlToken: controlToken,
httpClient: &clientCopy,
}
}
func (c *runtimeAgentHTTPClient) Health(ctx context.Context, endpoint string) error {
return c.do(ctx, http.MethodGet, endpoint, "/v1/health", nil, nil)
}
func (c *runtimeAgentHTTPClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) {
var resp RuntimeAgentCreateGatewayResponse
if err := c.do(ctx, http.MethodPost, endpoint, "/v1/gateways", req, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func (c *runtimeAgentHTTPClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error {
return c.do(ctx, http.MethodDelete, endpoint, "/v1/gateways/"+url.PathEscape(gatewayID), nil, nil)
}
func (c *runtimeAgentHTTPClient) Drain(ctx context.Context, endpoint string) error {
return c.do(ctx, http.MethodPost, endpoint, "/v1/drain", map[string]bool{"draining": true}, nil)
}
func (c *runtimeAgentHTTPClient) do(ctx context.Context, method, endpoint, path string, body any, out any) error {
endpoint = strings.TrimRight(endpoint, "/")
var reader io.Reader
if body != nil {
payload, err := json.Marshal(body)
if err != nil {
return err
}
reader = bytes.NewReader(payload)
}
req, err := http.NewRequestWithContext(ctx, method, endpoint+path, reader)
if err != nil {
return err
}
req.Header.Set("X-ClawManager-Control-Token", c.controlToken)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
if resp.StatusCode == http.StatusConflict {
return fmt.Errorf("runtime agent conflict: %s", string(msg))
}
return fmt.Errorf("runtime agent status %d: %s", resp.StatusCode, string(msg))
}
if out == nil {
return nil
}
return json.NewDecoder(resp.Body).Decode(out)
}
backend/internal/services/runtime_agent_client_test.go
+223
View File
@@ -0,0 +1,223 @@
package services
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestRuntimeAgentClientCreateGateway(t *testing.T) {
var gotToken string
var gotReq RuntimeAgentCreateGatewayRequest
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/gateways" {
t.Fatalf("unexpected path %s", r.URL.Path)
}
gotToken = r.Header.Get("X-ClawManager-Control-Token")
if err := json.NewDecoder(r.Body).Decode(&gotReq); err != nil {
t.Fatalf("decode request: %v", err)
}
_ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{
GatewayID: "gw-7-3",
Port: 20017,
PID: runtimeAgentIntPtr(8842),
Status: "running",
})
}))
defer server.Close()
client := NewRuntimeAgentClient("secret")
resp, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{
InstanceID: 7,
UserID: 8,
AgentType: "openclaw",
WorkspacePath: "/workspaces/openclaw/user-8/instance-7",
PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099},
UID: 200007,
GID: 200007,
CPUCores: 2,
MemoryMB: 4096,
DiskQuotaMB: 20480,
Generation: 3,
})
if err != nil {
t.Fatalf("CreateGateway returned error: %v", err)
}
if gotToken != "secret" {
t.Fatalf("unexpected token %q", gotToken)
}
if gotReq.InstanceID != 7 || gotReq.PortRange.Start != 20000 {
t.Fatalf("unexpected request %#v", gotReq)
}
if resp.GatewayID != "gw-7-3" || resp.Port != 20017 {
t.Fatalf("unexpected response %#v", resp)
}
}
func TestRuntimeAgentClientCreateGatewayTrimsEndpointSlash(t *testing.T) {
var calls int
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
if r.URL.Path != "/v1/gateways" {
t.Fatalf("unexpected path %s", r.URL.Path)
}
_ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{
GatewayID: "gw-7-3",
Port: 20017,
Status: "running",
})
}))
defer server.Close()
client := NewRuntimeAgentClient("secret")
if _, err := client.CreateGateway(context.Background(), server.URL+"/", RuntimeAgentCreateGatewayRequest{}); err != nil {
t.Fatalf("CreateGateway returned error: %v", err)
}
if calls != 1 {
t.Fatalf("expected one gateway create call, got %d", calls)
}
}
func TestRuntimeAgentClientDefaultTimeoutAllowsGatewayStartup(t *testing.T) {
client := NewRuntimeAgentClient("secret")
agentClient, ok := client.(*runtimeAgentHTTPClient)
if !ok {
t.Fatalf("unexpected client type %T", client)
}
if agentClient.httpClient.Timeout != 30*time.Second {
t.Fatalf("default timeout = %v, want 30s", agentClient.httpClient.Timeout)
}
}
func TestRuntimeAgentClientDeleteGatewayEscapesGatewayID(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
t.Fatalf("unexpected method %s", r.Method)
}
if r.URL.EscapedPath() != "/v1/gateways/gw-7%2F3" {
t.Fatalf("unexpected escaped path %s", r.URL.EscapedPath())
}
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
client := NewRuntimeAgentClient("secret")
if err := client.DeleteGateway(context.Background(), server.URL, "gw-7/3"); err != nil {
t.Fatalf("DeleteGateway returned error: %v", err)
}
}
func TestRuntimeAgentClientHealthHandlesSuccessAndRedirect(t *testing.T) {
successServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/health" {
t.Fatalf("unexpected success path %s", r.URL.Path)
}
w.WriteHeader(http.StatusNoContent)
}))
defer successServer.Close()
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1/health":
w.Header().Set("Location", "/ok")
w.WriteHeader(http.StatusFound)
_, _ = w.Write([]byte("redirect not allowed"))
case "/ok":
w.WriteHeader(http.StatusOK)
default:
t.Fatalf("unexpected redirect path %s", r.URL.Path)
}
}))
defer redirectServer.Close()
client := NewRuntimeAgentClient("secret")
if err := client.Health(context.Background(), successServer.URL); err != nil {
t.Fatalf("Health returned error for 204: %v", err)
}
err := client.Health(context.Background(), redirectServer.URL)
if err == nil || !strings.Contains(err.Error(), "runtime agent status 302") || !strings.Contains(err.Error(), "redirect not allowed") {
t.Fatalf("unexpected redirect error %v", err)
}
}
func TestRuntimeAgentClientWithHTTPClientDoesNotMutateRedirectPolicy(t *testing.T) {
supplied := &http.Client{Timeout: 10 * time.Second}
client := NewRuntimeAgentClientWithHTTPClient("secret", supplied)
if supplied.CheckRedirect != nil {
t.Fatalf("constructor mutated supplied CheckRedirect")
}
agentClient, ok := client.(*runtimeAgentHTTPClient)
if !ok {
t.Fatalf("unexpected client type %T", client)
}
if agentClient.httpClient == supplied {
t.Fatalf("constructor reused supplied client pointer")
}
if agentClient.httpClient.Timeout != supplied.Timeout {
t.Fatalf("timeout was not preserved: got %v want %v", agentClient.httpClient.Timeout, supplied.Timeout)
}
if agentClient.httpClient.CheckRedirect == nil {
t.Fatalf("copied client has no redirect policy")
}
}
func TestRuntimeAgentClientDrainSendsJSONBody(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/drain" {
t.Fatalf("unexpected path %s", r.URL.Path)
}
if r.Header.Get("Content-Type") != "application/json" {
t.Fatalf("unexpected content type %q", r.Header.Get("Content-Type"))
}
body, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read body: %v", err)
}
if strings.TrimSpace(string(body)) != `{"draining":true}` {
t.Fatalf("unexpected body %s", body)
}
w.WriteHeader(http.StatusAccepted)
}))
defer server.Close()
client := NewRuntimeAgentClient("secret")
if err := client.Drain(context.Background(), server.URL); err != nil {
t.Fatalf("Drain returned error: %v", err)
}
}
func TestRuntimeAgentClientConflict(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "no free port", http.StatusConflict)
}))
defer server.Close()
client := NewRuntimeAgentClient("secret")
_, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{})
if err == nil || !strings.Contains(err.Error(), "runtime agent conflict") || !strings.Contains(err.Error(), "no free port") {
t.Fatalf("unexpected error %v", err)
}
}
func TestRuntimeAgentClientNonConflictErrorIncludesStatusAndBody(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "agent exploded", http.StatusInternalServerError)
}))
defer server.Close()
client := NewRuntimeAgentClient("secret")
_, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{})
if err == nil || !strings.Contains(err.Error(), "runtime agent status 500") || !strings.Contains(err.Error(), "agent exploded") {
t.Fatalf("unexpected error %v", err)
}
}
func runtimeAgentIntPtr(v int) *int {
return &v
}
backend/internal/services/runtime_capacity.go
+80
View File
@@ -0,0 +1,80 @@
package services
import (
"fmt"
"path"
"strings"
)
const (
RuntimeTypeOpenClaw = "openclaw"
RuntimeTypeHermes = "hermes"
InstanceModeLite = "lite"
InstanceModePro = "pro"
RuntimeBackendGateway = "gateway"
RuntimeBackendDesktop = "desktop"
RuntimeBackendShell = "shell"
RuntimeGatewayPortStart = 20000
RuntimeGatewayPortEnd = 20099
RuntimePodCapacity = 100
RuntimeLinuxIDBase = 200000
)
func NormalizeV2RuntimeType(instanceType string) (string, bool) {
switch strings.ToLower(strings.TrimSpace(instanceType)) {
case RuntimeTypeOpenClaw:
return RuntimeTypeOpenClaw, true
case RuntimeTypeHermes:
return RuntimeTypeHermes, true
default:
return "", false
}
}
func NormalizeInstanceMode(mode string) (string, bool) {
switch strings.ToLower(strings.TrimSpace(mode)) {
case InstanceModeLite:
return InstanceModeLite, true
case InstanceModePro:
return InstanceModePro, true
default:
return "", false
}
}
func RuntimeTypeForInstanceMode(mode string) (string, bool) {
normalized, ok := NormalizeInstanceMode(mode)
if !ok {
return "", false
}
if normalized == InstanceModeLite {
return RuntimeBackendGateway, true
}
return RuntimeBackendDesktop, true
}
func InstanceModeForRuntimeType(runtimeType string) string {
if strings.EqualFold(strings.TrimSpace(runtimeType), RuntimeBackendGateway) {
return InstanceModeLite
}
return InstanceModePro
}
func RuntimeWorkspacePath(runtimeType string, userID int, instanceID int) string {
return RuntimeWorkspacePathWithRoot("/workspaces", runtimeType, userID, instanceID)
}
func RuntimeWorkspacePathWithRoot(root, runtimeType string, userID int, instanceID int) string {
root = strings.TrimSpace(root)
if root == "" {
root = "/workspaces"
}
return path.Join(root, fmt.Sprintf("%s/user-%d/instance-%d", runtimeType, userID, instanceID))
}
func RuntimeLinuxID(instanceID int) int {
return RuntimeLinuxIDBase + instanceID
}
backend/internal/services/runtime_capacity_test.go
+52
View File
@@ -0,0 +1,52 @@
package services
import "testing"
func TestNormalizeV2RuntimeTypeAcceptsManagedRuntimeTypes(t *testing.T) {
for _, input := range []string{"openclaw", " OpenClaw ", "hermes", "Hermes"} {
got, ok := NormalizeV2RuntimeType(input)
if !ok {
t.Fatalf("expected %q to be accepted", input)
}
if got != RuntimeTypeOpenClaw && got != RuntimeTypeHermes {
t.Fatalf("expected normalized managed runtime type, got %q", got)
}
}
}
func TestNormalizeV2RuntimeTypeRejectsLegacyRuntimeTypes(t *testing.T) {
for _, input := range []string{"webtop", "ubuntu", "", "desktop", "shell"} {
if got, ok := NormalizeV2RuntimeType(input); ok {
t.Fatalf("expected %q to be rejected, got %q", input, got)
}
}
}
func TestRuntimeWorkspacePath(t *testing.T) {
got := RuntimeWorkspacePath("openclaw", 45, 123)
want := "/workspaces/openclaw/user-45/instance-123"
if got != want {
t.Fatalf("expected workspace path %q, got %q", want, got)
}
}
func TestRuntimeLinuxID(t *testing.T) {
if got, want := RuntimeLinuxID(123), 200123; got != want {
t.Fatalf("expected linux id %d, got %d", want, got)
}
}
func TestInstanceModeRuntimeTypeMapping(t *testing.T) {
if got, ok := RuntimeTypeForInstanceMode(" lite "); !ok || got != RuntimeBackendGateway {
t.Fatalf("lite runtime type = %q/%v, want gateway/true", got, ok)
}
if got, ok := RuntimeTypeForInstanceMode("Pro"); !ok || got != RuntimeBackendDesktop {
t.Fatalf("pro runtime type = %q/%v, want desktop/true", got, ok)
}
if got := InstanceModeForRuntimeType(RuntimeBackendGateway); got != InstanceModeLite {
t.Fatalf("gateway mode = %q, want lite", got)
}
if got := InstanceModeForRuntimeType(RuntimeBackendDesktop); got != InstanceModePro {
t.Fatalf("desktop mode = %q, want pro", got)
}
}
backend/internal/services/runtime_events.go
+68
View File
@@ -0,0 +1,68 @@
package services
import (
"context"
"encoding/json"
"time"
)
const runtimeEventStreamKey = "clawmanager:runtime-events"
type RuntimeEvent struct {
Type string `json:"type"`
Payload json.RawMessage `json:"payload"`
CreatedAt time.Time `json:"created_at"`
}
type RuntimeEventService interface {
Publish(ctx context.Context, eventType string, payload any) error
Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error)
}
func NewRuntimeEventService(redis PlatformRedisClient) RuntimeEventService {
if redis == nil {
return noopRuntimeEventService{}
}
return &redisRuntimeEventService{redis: redis}
}
type redisRuntimeEventService struct {
redis PlatformRedisClient
}
func (s *redisRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error {
payloadJSON, err := json.Marshal(payload)
if err != nil {
return err
}
event := RuntimeEvent{
Type: eventType,
Payload: payloadJSON,
CreatedAt: time.Now().UTC(),
}
eventJSON, err := json.Marshal(event)
if err != nil {
return err
}
_, err = s.redis.XAdd(ctx, runtimeEventStreamKey, map[string]string{
"type": event.Type,
"payload": string(event.Payload),
"created_at": event.CreatedAt.Format(time.RFC3339Nano),
"event": string(eventJSON),
})
return err
}
func (s *redisRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) {
return s.redis.XRead(ctx, runtimeEventStreamKey, lastID, block)
}
type noopRuntimeEventService struct{}
func (noopRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error {
return nil
}
func (noopRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) {
return nil, nil
}
backend/internal/services/runtime_events_test.go
+75
View File
@@ -0,0 +1,75 @@
package services
import (
"context"
"encoding/json"
"testing"
"time"
)
func TestRuntimeEventServiceNoopWhenRedisNil(t *testing.T) {
service := NewRuntimeEventService(nil)
if err := service.Publish(context.Background(), "runtime.test", map[string]string{"ok": "true"}); err != nil {
t.Fatalf("Publish returned error: %v", err)
}
messages, err := service.Read(context.Background(), "0", time.Millisecond)
if err != nil {
t.Fatalf("Read returned error: %v", err)
}
if messages != nil {
t.Fatalf("messages = %#v, want nil", messages)
}
}
func TestRuntimeEventServicePublishesJSONFields(t *testing.T) {
redis := &fakePlatformRedisClient{}
service := NewRuntimeEventService(redis)
if err := service.Publish(context.Background(), "runtime.instance.running", map[string]int{"instance_id": 17}); err != nil {
t.Fatalf("Publish returned error: %v", err)
}
if redis.xaddKey != runtimeEventStreamKey {
t.Fatalf("XAdd key = %q", redis.xaddKey)
}
if redis.xaddFields["type"] != "runtime.instance.running" {
t.Fatalf("type field = %q", redis.xaddFields["type"])
}
var payload map[string]int
if err := json.Unmarshal([]byte(redis.xaddFields["payload"]), &payload); err != nil {
t.Fatalf("payload is not json: %v", err)
}
if payload["instance_id"] != 17 {
t.Fatalf("payload instance_id = %d", payload["instance_id"])
}
var event RuntimeEvent
if err := json.Unmarshal([]byte(redis.xaddFields["event"]), &event); err != nil {
t.Fatalf("event is not json: %v", err)
}
if event.Type != "runtime.instance.running" {
t.Fatalf("event type = %q", event.Type)
}
}
type fakePlatformRedisClient struct {
xaddKey string
xaddFields map[string]string
}
func (c *fakePlatformRedisClient) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) {
return true, nil
}
func (c *fakePlatformRedisClient) Del(ctx context.Context, key string) error {
return nil
}
func (c *fakePlatformRedisClient) XAdd(ctx context.Context, key string, fields map[string]string) (string, error) {
c.xaddKey = key
c.xaddFields = fields
return "1-0", nil
}
func (c *fakePlatformRedisClient) XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) {
return []redisStreamMessage{{ID: "1-0", Fields: map[string]string{"type": "runtime.test"}}}, nil
}
backend/internal/services/runtime_leader.go
+99
View File
@@ -0,0 +1,99 @@
package services
import (
"context"
"time"
coordinationv1 "k8s.io/api/coordination/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
const (
runtimeLeaderLeaseName = "clawmanager-runtime-scheduler"
runtimeLeaderLeaseTTL = 15 * time.Second
)
type RuntimeLeaderService interface {
IsLeader(ctx context.Context) bool
}
type runtimeLeaderService struct {
client kubernetes.Interface
namespace string
holder string
}
func NewRuntimeLeaderService(client kubernetes.Interface, namespace, holder string) RuntimeLeaderService {
return &runtimeLeaderService{
client: client,
namespace: namespace,
holder: holder,
}
}
func (s *runtimeLeaderService) IsLeader(ctx context.Context) bool {
if s == nil || s.client == nil || s.namespace == "" || s.holder == "" {
return false
}
leases := s.client.CoordinationV1().Leases(s.namespace)
now := metav1.NewMicroTime(time.Now())
lease, err := leases.Get(ctx, runtimeLeaderLeaseName, metav1.GetOptions{})
if errors.IsNotFound(err) {
_, createErr := leases.Create(ctx, s.newLease(now), metav1.CreateOptions{})
if createErr == nil {
return true
}
if !errors.IsAlreadyExists(createErr) {
return false
}
lease, err = leases.Get(ctx, runtimeLeaderLeaseName, metav1.GetOptions{})
}
if err != nil {
return false
}
if !s.canAcquire(lease, now.Time) {
return false
}
lease.Spec.HolderIdentity = &s.holder
lease.Spec.RenewTime = &now
ttlSeconds := int32(runtimeLeaderLeaseTTL / time.Second)
lease.Spec.LeaseDurationSeconds = &ttlSeconds
if _, err := leases.Update(ctx, lease, metav1.UpdateOptions{}); err != nil {
return false
}
return true
}
func (s *runtimeLeaderService) newLease(now metav1.MicroTime) *coordinationv1.Lease {
ttlSeconds := int32(runtimeLeaderLeaseTTL / time.Second)
return &coordinationv1.Lease{
ObjectMeta: metav1.ObjectMeta{
Name: runtimeLeaderLeaseName,
Namespace: s.namespace,
},
Spec: coordinationv1.LeaseSpec{
HolderIdentity: &s.holder,
AcquireTime: &now,
RenewTime: &now,
LeaseDurationSeconds: &ttlSeconds,
},
}
}
func (s *runtimeLeaderService) canAcquire(lease *coordinationv1.Lease, now time.Time) bool {
if lease == nil {
return true
}
if lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity == s.holder {
return true
}
if lease.Spec.RenewTime == nil {
return true
}
return lease.Spec.RenewTime.Add(runtimeLeaderLeaseTTL).Before(now)
}
backend/internal/services/runtime_leader_test.go
+67
View File
@@ -0,0 +1,67 @@
package services
import (
"context"
"testing"
"time"
coordinationv1 "k8s.io/api/coordination/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
func TestRuntimeLeaderServiceFirstCallerBecomesLeader(t *testing.T) {
service := NewRuntimeLeaderService(fake.NewSimpleClientset(), "runtime-system", "backend-a")
if !service.IsLeader(context.Background()) {
t.Fatalf("expected first caller to become leader")
}
}
func TestRuntimeLeaderServiceDifferentCallerCannotStealUnexpiredLease(t *testing.T) {
now := metav1.NewMicroTime(time.Now())
holder := "backend-a"
client := fake.NewSimpleClientset(&coordinationv1.Lease{
ObjectMeta: metav1.ObjectMeta{Name: "clawmanager-runtime-scheduler", Namespace: "runtime-system"},
Spec: coordinationv1.LeaseSpec{
HolderIdentity: &holder,
RenewTime: &now,
LeaseDurationSeconds: int32Ptr(15),
},
})
service := NewRuntimeLeaderService(client, "runtime-system", "backend-b")
if service.IsLeader(context.Background()) {
t.Fatalf("expected different caller not to steal unexpired lease")
}
}
func TestRuntimeLeaderServiceDifferentCallerCanAcquireExpiredLease(t *testing.T) {
renewTime := metav1.NewMicroTime(time.Now().Add(-30 * time.Second))
holder := "backend-a"
client := fake.NewSimpleClientset(&coordinationv1.Lease{
ObjectMeta: metav1.ObjectMeta{Name: "clawmanager-runtime-scheduler", Namespace: "runtime-system"},
Spec: coordinationv1.LeaseSpec{
HolderIdentity: &holder,
RenewTime: &renewTime,
LeaseDurationSeconds: int32Ptr(15),
},
})
service := NewRuntimeLeaderService(client, "runtime-system", "backend-b")
if !service.IsLeader(context.Background()) {
t.Fatalf("expected different caller to acquire expired lease")
}
lease, err := client.CoordinationV1().Leases("runtime-system").Get(context.Background(), "clawmanager-runtime-scheduler", metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to load lease: %v", err)
}
if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != "backend-b" {
t.Fatalf("expected lease holder backend-b, got %#v", lease.Spec.HolderIdentity)
}
}
func int32Ptr(value int32) *int32 {
return &value
}
backend/internal/services/runtime_scheduler.go
+880
View File
@@ -0,0 +1,880 @@
package services
import (
"context"
"errors"
"fmt"
"log"
"strconv"
"strings"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services/k8s"
)
type RuntimeScheduler struct {
instanceRepo repository.InstanceRepository
podRepo repository.RuntimePodRepository
bindingRepo repository.InstanceRuntimeBindingRepository
rolloutRepo repository.RuntimeRolloutRepository
agentClient RuntimeAgentClient
events RuntimeEventService
leader RuntimeLeaderService
deployments k8s.RuntimeDeploymentService
envBuilder RuntimeGatewayEnvBuilder
tick time.Duration
workspaceRoot string
runtimeNamespace string
gatewayPortStart int
gatewayPortEnd int
heartbeatTimeout time.Duration
maxGatewaysPerPod int
}
var errRuntimeScaleOutPending = errors.New("runtime scale-out pending")
const runtimeRolloutStaleWindowMultiplier = 3
type RuntimeGatewayEnvBuilder func(*models.Instance) (map[string]string, error)
type RuntimeSchedulerOption func(*RuntimeScheduler)
func WithRuntimeSchedulerWorkspaceRoot(root string) RuntimeSchedulerOption {
return func(s *RuntimeScheduler) {
if strings.TrimSpace(root) != "" {
s.workspaceRoot = strings.TrimSpace(root)
}
}
}
func WithRuntimeSchedulerGatewayPortRange(start, end int) RuntimeSchedulerOption {
return func(s *RuntimeScheduler) {
if start > 0 {
s.gatewayPortStart = start
}
if end > 0 {
s.gatewayPortEnd = end
}
if s.gatewayPortEnd < s.gatewayPortStart {
s.gatewayPortEnd = s.gatewayPortStart
}
}
}
func WithRuntimeSchedulerHeartbeatTimeout(timeout time.Duration) RuntimeSchedulerOption {
return func(s *RuntimeScheduler) {
s.heartbeatTimeout = timeout
}
}
func WithRuntimeSchedulerNamespace(namespace string) RuntimeSchedulerOption {
return func(s *RuntimeScheduler) {
if strings.TrimSpace(namespace) != "" {
s.runtimeNamespace = strings.TrimSpace(namespace)
}
}
}
func WithRuntimeSchedulerMaxGatewaysPerPod(capacity int) RuntimeSchedulerOption {
return func(s *RuntimeScheduler) {
if capacity > 0 {
s.maxGatewaysPerPod = capacity
}
}
}
func WithRuntimeSchedulerGatewayEnvBuilder(builder RuntimeGatewayEnvBuilder) RuntimeSchedulerOption {
return func(s *RuntimeScheduler) {
s.envBuilder = builder
}
}
func NewRuntimeScheduler(
instanceRepo repository.InstanceRepository,
podRepo repository.RuntimePodRepository,
bindingRepo repository.InstanceRuntimeBindingRepository,
rolloutRepo repository.RuntimeRolloutRepository,
agentClient RuntimeAgentClient,
events RuntimeEventService,
leader RuntimeLeaderService,
deployments k8s.RuntimeDeploymentService,
tick time.Duration,
opts ...RuntimeSchedulerOption,
) *RuntimeScheduler {
if tick <= 0 {
tick = 2 * time.Second
}
s := &RuntimeScheduler{
instanceRepo: instanceRepo,
podRepo: podRepo,
bindingRepo: bindingRepo,
rolloutRepo: rolloutRepo,
agentClient: agentClient,
events: events,
leader: leader,
deployments: deployments,
tick: tick,
workspaceRoot: "/workspaces",
runtimeNamespace: "clawmanager-system",
gatewayPortStart: RuntimeGatewayPortStart,
gatewayPortEnd: RuntimeGatewayPortEnd,
heartbeatTimeout: 10 * time.Second,
maxGatewaysPerPod: RuntimePodCapacity,
}
for _, opt := range opts {
opt(s)
}
return s
}
func (s *RuntimeScheduler) Start(ctx context.Context) {
if s == nil {
return
}
go func() {
ticker := time.NewTicker(s.tick)
defer ticker.Stop()
s.reconcileIfLeader(ctx)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.reconcileIfLeader(ctx)
}
}
}()
}
func (s *RuntimeScheduler) HeartbeatTimeout() time.Duration {
if s == nil {
return 0
}
return s.heartbeatTimeout
}
func (s *RuntimeScheduler) DrainPod(ctx context.Context, podID int64) error {
if s == nil || s.podRepo == nil {
return fmt.Errorf("runtime scheduler pod repository is not configured")
}
pod, err := s.podRepo.GetByID(ctx, podID)
if err != nil {
return err
}
if pod == nil {
return fmt.Errorf("runtime pod %d not found", podID)
}
if pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" && s.agentClient != nil {
if err := s.agentClient.Drain(ctx, strings.TrimSpace(*pod.AgentEndpoint)); err != nil {
return err
}
}
return s.podRepo.MarkState(ctx, podID, "draining", true)
}
func (s *RuntimeScheduler) FailoverPod(ctx context.Context, podID int64, reason string) error {
if s == nil || s.podRepo == nil || s.bindingRepo == nil || s.instanceRepo == nil {
return fmt.Errorf("runtime scheduler dependencies are not configured")
}
var errs []error
if err := s.podRepo.MarkState(ctx, podID, "unhealthy", false); err != nil {
errs = append(errs, err)
}
bindings, err := s.bindingRepo.ListByRuntimePodID(ctx, podID)
if err != nil {
errs = append(errs, err)
return errors.Join(errs...)
}
for _, binding := range bindings {
nextGeneration := binding.Generation + 1
instance, err := s.instanceRepo.GetByID(binding.InstanceID)
if err != nil {
errs = append(errs, err)
continue
}
if instance != nil {
nextGeneration = instance.RuntimeGeneration + 1
}
message := reason
if err := s.instanceRepo.UpdateRuntimeState(ctx, binding.InstanceID, "creating", nextGeneration, &message); err != nil {
errs = append(errs, err)
continue
}
if err := s.bindingRepo.DeleteByInstanceID(ctx, binding.InstanceID); err != nil {
errs = append(errs, err)
continue
}
if err := s.podRepo.ReleaseSlot(ctx, podID); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
func (s *RuntimeScheduler) StartRollout(ctx context.Context, rolloutID int64) error {
if s == nil || s.rolloutRepo == nil || s.podRepo == nil {
return fmt.Errorf("runtime scheduler rollout dependencies are not configured")
}
rollout, err := s.rolloutRepo.GetByID(ctx, rolloutID)
if err != nil {
return err
}
if rollout == nil {
return fmt.Errorf("runtime rollout %d not found", rolloutID)
}
startedAt := time.Now().UTC()
if err := s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "running", &startedAt, nil, nil); err != nil {
return err
}
allPods, err := s.podRepo.List(ctx, rollout.RuntimeType)
if err != nil {
message := err.Error()
_ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message)
return err
}
currentPods := s.currentRuntimePods(allPods, time.Now().UTC())
if runtimePodsAlreadyAtImage(currentPods, rollout.RuntimeType, rollout.TargetImageRef) {
finishedAt := time.Now().UTC()
return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil)
}
batchSize := rollout.BatchSize
if batchSize <= 0 {
batchSize = 1
}
maxUnavailable := rollout.MaxUnavailable
if maxUnavailable <= 0 {
maxUnavailable = 1
}
if err := s.rolloutRuntimeDeployments(ctx, rollout, allPods, maxUnavailable, batchSize); err != nil {
message := err.Error()
_ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message)
return err
}
unavailable := 0
readyCandidates := 0
for _, pod := range currentPods {
if pod.Draining || pod.State != "ready" {
unavailable++
continue
}
readyCandidates++
}
remaining := maxUnavailable - unavailable
if remaining <= 0 {
return nil
}
drainLimit := minInt(batchSize, remaining)
var errs []error
drained := 0
for _, pod := range currentPods {
if drained >= drainLimit {
break
}
if pod.State != "ready" || pod.Draining {
continue
}
if err := s.DrainPod(ctx, pod.ID); err != nil {
errs = append(errs, err)
continue
}
drained++
}
if len(errs) > 0 {
joined := errors.Join(errs...)
message := joined.Error()
_ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message)
return joined
}
if drained == 0 && readyCandidates == 0 && unavailable == 0 && len(currentPods) > 0 {
finishedAt := time.Now().UTC()
return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil)
}
return nil
}
func (s *RuntimeScheduler) reconcileRollouts(ctx context.Context) error {
if s == nil || s.rolloutRepo == nil {
return nil
}
rollouts, err := s.rolloutRepo.ListActive(ctx, "")
if err != nil {
return fmt.Errorf("list active runtime rollouts: %w", err)
}
var errs []error
for _, rollout := range rollouts {
switch rollout.Status {
case "pending":
if err := s.StartRollout(ctx, rollout.ID); err != nil {
errs = append(errs, fmt.Errorf("start runtime rollout %d: %w", rollout.ID, err))
}
case "running":
if err := s.finishRolloutIfReady(ctx, rollout); err != nil {
errs = append(errs, fmt.Errorf("finish runtime rollout %d: %w", rollout.ID, err))
}
}
}
return errors.Join(errs...)
}
func (s *RuntimeScheduler) rolloutRuntimeDeployments(ctx context.Context, rollout *models.RuntimeRollout, pods []models.RuntimePod, maxUnavailable, maxSurge int) error {
if s == nil || s.deployments == nil || rollout == nil {
return nil
}
targetImage := strings.TrimSpace(rollout.TargetImageRef)
if targetImage == "" {
return nil
}
type deploymentRef struct {
namespace string
name string
}
refs := map[deploymentRef]struct{}{}
for _, pod := range pods {
if pod.RuntimeType != rollout.RuntimeType {
continue
}
namespace := strings.TrimSpace(pod.Namespace)
name := strings.TrimSpace(pod.DeploymentName)
if namespace == "" || name == "" {
continue
}
refs[deploymentRef{namespace: namespace, name: name}] = struct{}{}
}
if len(refs) == 0 {
name := defaultRuntimeDeploymentName(rollout.RuntimeType)
if name == "" {
return fmt.Errorf("no runtime deployment found for %s rollout", rollout.RuntimeType)
}
refs[deploymentRef{namespace: s.runtimeNamespace, name: name}] = struct{}{}
}
var errs []error
for ref := range refs {
if err := s.deployments.RolloutImage(ctx, ref.namespace, ref.name, targetImage, maxUnavailable, maxSurge); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
func (s *RuntimeScheduler) RuntimeDeploymentPods(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
if s == nil || s.deployments == nil {
return nil, nil
}
runtimeType = strings.TrimSpace(runtimeType)
pods, err := s.deployments.ListPods(ctx, s.runtimeNamespace, runtimeType)
if err != nil {
return nil, err
}
result := make([]models.RuntimePod, 0, len(pods))
for index, pod := range pods {
result = append(result, models.RuntimePod{
ID: -int64(index + 1),
RuntimeType: pod.RuntimeType,
Namespace: pod.Namespace,
PodName: pod.PodName,
PodIP: pod.PodIP,
NodeName: pod.NodeName,
DeploymentName: pod.DeploymentName,
ImageRef: pod.ImageRef,
State: runtimeDeploymentFallbackState(pod.State),
Capacity: s.maxGatewaysPerPod,
UsedSlots: 0,
Draining: false,
})
}
return result, nil
}
func defaultRuntimeDeploymentName(runtimeType string) string {
switch strings.ToLower(strings.TrimSpace(runtimeType)) {
case RuntimeTypeOpenClaw:
return "openclaw-runtime"
case RuntimeTypeHermes:
return "hermes-runtime"
default:
return ""
}
}
func runtimeDeploymentFallbackState(k8sState string) string {
switch strings.TrimSpace(k8sState) {
case "pending", "deleted":
return strings.TrimSpace(k8sState)
default:
return "unhealthy"
}
}
func (s *RuntimeScheduler) finishRolloutIfReady(ctx context.Context, rollout models.RuntimeRollout) error {
if s == nil || s.podRepo == nil || s.rolloutRepo == nil {
return nil
}
targetImage := strings.TrimSpace(rollout.TargetImageRef)
if targetImage == "" {
return nil
}
pods, err := s.podRepo.List(ctx, rollout.RuntimeType)
if err != nil {
return err
}
pods = s.currentRuntimePods(pods, time.Now().UTC())
if len(pods) == 0 {
return nil
}
for _, pod := range pods {
if pod.RuntimeType != rollout.RuntimeType {
continue
}
if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage {
return nil
}
}
finishedAt := time.Now().UTC()
return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", rollout.StartedAt, &finishedAt, nil)
}
func (s *RuntimeScheduler) currentRuntimePods(pods []models.RuntimePod, now time.Time) []models.RuntimePod {
if s == nil || s.heartbeatTimeout <= 0 {
return pods
}
cutoff := now.UTC().Add(-runtimeRolloutStaleWindowMultiplier * s.heartbeatTimeout)
current := pods[:0]
for _, pod := range pods {
if pod.LastSeenAt != nil && pod.LastSeenAt.UTC().Before(cutoff) {
continue
}
current = append(current, pod)
}
return current
}
func runtimePodsAlreadyAtImage(pods []models.RuntimePod, runtimeType, targetImage string) bool {
targetImage = strings.TrimSpace(targetImage)
if targetImage == "" {
return false
}
matched := 0
for _, pod := range pods {
if pod.RuntimeType != runtimeType {
continue
}
matched++
if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage {
return false
}
}
return matched > 0
}
func (s *RuntimeScheduler) reconcileIfLeader(ctx context.Context) {
if s.leader != nil && !s.leader.IsLeader(ctx) {
return
}
if err := s.reconcile(ctx); err != nil {
log.Printf("runtime scheduler reconcile failed: %v", err)
}
}
func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
if s == nil || s.instanceRepo == nil || s.bindingRepo == nil || s.podRepo == nil {
return fmt.Errorf("runtime scheduler dependencies are not configured")
}
var errs []error
if err := s.failoverStalePods(ctx); err != nil {
errs = append(errs, err)
}
if err := s.reconcileRollouts(ctx); err != nil {
errs = append(errs, err)
}
creating, err := s.instanceRepo.GetV2Creating(ctx, 100)
if err != nil {
errs = append(errs, err)
} else {
for _, instance := range creating {
if !isSchedulerManagedV2Instance(instance) {
continue
}
binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
if err != nil {
errs = append(errs, fmt.Errorf("get binding for creating instance %d: %w", instance.ID, err))
continue
}
if binding != nil {
continue
}
if err := s.assignInstance(ctx, instance); err != nil {
if errors.Is(err, errRuntimeScaleOutPending) {
continue
}
errs = append(errs, fmt.Errorf("assign creating instance %d: %w", instance.ID, err))
s.markInstanceError(ctx, instance, err, &errs)
}
}
}
desired, err := s.instanceRepo.GetV2DesiredRunning(ctx, 100)
if err != nil {
errs = append(errs, err)
} else {
for _, instance := range desired {
if !isSchedulerManagedV2Instance(instance) {
continue
}
if isRuntimeErrorInstance(instance) && !isRecoverableRuntimeSchedulingError(instance) {
continue
}
binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID)
if err != nil {
errs = append(errs, fmt.Errorf("get running binding for instance %d: %w", instance.ID, err))
continue
}
if binding != nil {
continue
}
binding, err = s.bindingRepo.GetByInstanceID(ctx, instance.ID)
if err != nil {
errs = append(errs, fmt.Errorf("get binding for desired instance %d: %w", instance.ID, err))
continue
}
if binding != nil {
continue
}
if assignErr := s.assignInstance(ctx, instance); assignErr != nil {
if errors.Is(assignErr, errRuntimeScaleOutPending) {
continue
}
errs = append(errs, fmt.Errorf("assign desired instance %d: %w", instance.ID, assignErr))
s.markInstanceError(ctx, instance, assignErr, &errs)
}
}
}
return errors.Join(errs...)
}
func (s *RuntimeScheduler) failoverStalePods(ctx context.Context) error {
if s.heartbeatTimeout <= 0 {
return nil
}
pods, err := s.podRepo.List(ctx, "")
if err != nil {
return fmt.Errorf("list runtime pods for heartbeat failover: %w", err)
}
cutoff := time.Now().UTC().Add(-s.heartbeatTimeout)
var errs []error
for _, pod := range pods {
if pod.State == "unhealthy" || pod.State == "pending" {
continue
}
if pod.LastSeenAt == nil || !pod.LastSeenAt.Before(cutoff) {
continue
}
reason := fmt.Sprintf("runtime pod heartbeat lost since %s", pod.LastSeenAt.UTC().Format(time.RFC3339))
if err := s.FailoverPod(ctx, pod.ID, reason); err != nil {
errs = append(errs, fmt.Errorf("failover stale runtime pod %d: %w", pod.ID, err))
}
}
return errors.Join(errs...)
}
func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error {
if s == nil || s.podRepo == nil {
return fmt.Errorf("runtime scheduler pod repository is not configured")
}
if !isSchedulerManagedV2Instance(instance) {
return fmt.Errorf("instance %d is not scheduler-managed v2", instance.ID)
}
if s.bindingRepo != nil {
binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
if err != nil {
return fmt.Errorf("failed to check existing binding: %w", err)
}
if binding != nil {
return nil
}
}
runtimeType, ok := schedulerRuntimeType(instance)
if !ok {
return fmt.Errorf("unsupported runtime type %q", instance.Type)
}
pods, err := s.podRepo.ListSchedulable(ctx, runtimeType)
if err != nil {
return err
}
if len(pods) == 0 {
scaled, err := s.scaleOutIfAtCapacity(ctx, runtimeType)
if err != nil {
return fmt.Errorf("scale out %s runtime deployment: %w", runtimeType, err)
}
if scaled {
return errRuntimeScaleOutPending
}
}
var lastErr error
for _, pod := range pods {
if pod.AgentEndpoint == nil || strings.TrimSpace(*pod.AgentEndpoint) == "" {
continue
}
claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID)
if err != nil {
lastErr = err
continue
}
if !claimed {
continue
}
if err := s.createGatewayOnPod(ctx, instance, runtimeType, pod); err != nil {
if releaseErr := s.podRepo.ReleaseSlot(ctx, pod.ID); releaseErr != nil {
return errors.Join(err, releaseErr)
}
lastErr = err
continue
}
return nil
}
if lastErr != nil {
return fmt.Errorf("no schedulable %s runtime pod: %w", runtimeType, lastErr)
}
return fmt.Errorf("no schedulable %s runtime pod", runtimeType)
}
type runtimeDeploymentCapacity struct {
namespace string
name string
active int
full int
}
func (s *RuntimeScheduler) scaleOutIfAtCapacity(ctx context.Context, runtimeType string) (bool, error) {
if s == nil || s.deployments == nil || s.podRepo == nil {
return false, nil
}
pods, err := s.podRepo.List(ctx, runtimeType)
if err != nil {
return false, fmt.Errorf("list %s runtime pods for scale-out: %w", runtimeType, err)
}
groups := map[string]*runtimeDeploymentCapacity{}
for _, pod := range pods {
if pod.RuntimeType != runtimeType || pod.State != "ready" || pod.Draining || pod.Capacity <= 0 {
continue
}
namespace := strings.TrimSpace(pod.Namespace)
name := strings.TrimSpace(pod.DeploymentName)
if namespace == "" || name == "" {
continue
}
key := namespace + "/" + name
group := groups[key]
if group == nil {
group = &runtimeDeploymentCapacity{namespace: namespace, name: name}
groups[key] = group
}
group.active++
if pod.UsedSlots >= pod.Capacity {
group.full++
}
}
var target *runtimeDeploymentCapacity
for _, group := range groups {
if group.active == 0 || group.active != group.full {
continue
}
if target == nil || group.active > target.active {
target = group
}
}
if target == nil {
return false, nil
}
replicas := int32(target.active + 1)
if err := s.deployments.Scale(ctx, target.namespace, target.name, replicas); err != nil {
return false, err
}
if s.events != nil {
if err := s.events.Publish(ctx, "runtime.pool.scaleout", map[string]any{
"runtime_type": runtimeType,
"namespace": target.namespace,
"deployment": target.name,
"replicas": replicas,
}); err != nil {
log.Printf("runtime scheduler publish scale-out event failed: %v", err)
}
}
return true, nil
}
func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance models.Instance, runtimeType string, pod models.RuntimePod) error {
if s.agentClient == nil || s.bindingRepo == nil || s.instanceRepo == nil {
return fmt.Errorf("runtime scheduler gateway dependencies are not configured")
}
endpoint := strings.TrimSpace(*pod.AgentEndpoint)
workspacePath := RuntimeWorkspacePathWithRoot(s.workspaceRoot, runtimeType, instance.UserID, instance.ID)
environment, err := s.gatewayEnvironment(&instance)
if err != nil {
return fmt.Errorf("build runtime gateway environment: %w", err)
}
uid, gid := runtimeGatewayLinuxIDs(instance.ID, environment)
resp, err := s.agentClient.CreateGateway(ctx, endpoint, RuntimeAgentCreateGatewayRequest{
InstanceID: instance.ID,
UserID: instance.UserID,
AgentType: runtimeType,
WorkspacePath: workspacePath,
PortRange: RuntimeAgentPortRange{
Start: s.gatewayPortStart,
End: s.gatewayPortEnd,
},
UID: uid,
GID: gid,
CPUCores: instance.CPUCores,
MemoryMB: instance.MemoryGB * 1024,
DiskQuotaMB: instance.DiskGB * 1024,
Generation: instance.RuntimeGeneration,
Environment: environment,
})
if err != nil {
return err
}
if resp == nil {
return fmt.Errorf("runtime agent returned empty gateway response")
}
now := time.Now().UTC()
binding := &models.InstanceRuntimeBinding{
InstanceID: instance.ID,
RuntimePodID: pod.ID,
RuntimeType: runtimeType,
GatewayID: resp.GatewayID,
GatewayPort: resp.Port,
GatewayPID: resp.PID,
WorkspacePath: workspacePath,
State: "running",
Generation: instance.RuntimeGeneration,
LastHealthAt: &now,
}
if err := s.bindingRepo.Create(ctx, binding); err != nil {
return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, false, err)
}
if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err)
}
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "running", instance.RuntimeGeneration, nil); err != nil {
return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err)
}
if s.events != nil {
if err := s.events.Publish(ctx, "runtime.instance.running", map[string]any{
"instance_id": instance.ID,
"runtime_type": runtimeType,
"runtime_pod_id": pod.ID,
"gateway_id": resp.GatewayID,
"gateway_port": resp.Port,
"workspace_path": workspacePath,
"generation": instance.RuntimeGeneration,
}); err != nil {
log.Printf("runtime scheduler publish event failed: %v", err)
}
}
return nil
}
func runtimeGatewayLinuxIDs(instanceID int, environment map[string]string) (int, int) {
linuxID := RuntimeLinuxID(instanceID)
if !strings.EqualFold(strings.TrimSpace(environment["CLAWMANAGER_TEAM_ENABLED"]), "true") {
return linuxID, linuxID
}
sharedGID, err := strconv.Atoi(strings.TrimSpace(environment["CLAWMANAGER_TEAM_SHARED_GID"]))
if err != nil || sharedGID <= 0 {
return linuxID, linuxID
}
return linuxID, sharedGID
}
func (s *RuntimeScheduler) gatewayEnvironment(instance *models.Instance) (map[string]string, error) {
if s == nil || s.envBuilder == nil {
return nil, nil
}
env, err := s.envBuilder(instance)
if err != nil {
return nil, err
}
if len(env) == 0 {
return nil, nil
}
copied := make(map[string]string, len(env))
for key, value := range env {
if strings.TrimSpace(key) != "" {
copied[key] = value
}
}
if len(copied) == 0 {
return nil, nil
}
return copied, nil
}
func (s *RuntimeScheduler) cleanupGatewayAfterAssignFailure(ctx context.Context, endpoint string, instanceID int, gatewayID string, bindingCreated bool, cause error) error {
errs := []error{cause}
if gatewayID != "" && s.agentClient != nil {
if err := s.agentClient.DeleteGateway(ctx, endpoint, gatewayID); err != nil {
errs = append(errs, fmt.Errorf("delete gateway %s: %w", gatewayID, err))
}
}
if bindingCreated && s.bindingRepo != nil {
if err := s.bindingRepo.DeleteByInstanceID(ctx, instanceID); err != nil {
errs = append(errs, fmt.Errorf("delete binding for instance %d: %w", instanceID, err))
}
}
return errors.Join(errs...)
}
func (s *RuntimeScheduler) markInstanceError(ctx context.Context, instance models.Instance, cause error, errs *[]error) {
message := cause.Error()
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &message); err != nil {
*errs = append(*errs, fmt.Errorf("mark instance %d error: %w", instance.ID, err))
}
}
func schedulerRuntimeType(instance models.Instance) (string, bool) {
return NormalizeV2RuntimeType(instance.Type)
}
func isSchedulerManagedV2Instance(instance models.Instance) bool {
if _, ok := schedulerRuntimeType(instance); !ok {
return false
}
if normalizeInstanceRuntimeType(instance.RuntimeType) != RuntimeBackendGateway {
return false
}
if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok && mode != InstanceModeLite {
return false
}
return instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != ""
}
func isRuntimeErrorInstance(instance models.Instance) bool {
return strings.EqualFold(strings.TrimSpace(instance.Status), "error")
}
func isRecoverableRuntimeSchedulingError(instance models.Instance) bool {
if !isRuntimeErrorInstance(instance) || instance.RuntimeErrorMessage == nil {
return false
}
runtimeType, ok := schedulerRuntimeType(instance)
if !ok {
return false
}
message := strings.TrimSpace(*instance.RuntimeErrorMessage)
return message == fmt.Sprintf("no schedulable %s runtime pod", runtimeType)
}
func minInt(a, b int) int {
if a < b {
return a
}
return b
}
backend/internal/services/runtime_scheduler_test.go
+1840
View File
File diff suppressed because it is too large Load Diff
backend/internal/services/runtime_workspace_file_service.go
+660
View File
@@ -0,0 +1,660 @@
package services
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"os"
"path/filepath"
"sort"
"strings"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services/k8s"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/remotecommand"
)
const (
runtimeWorkspaceErrPrefix = "CLAW_WORKSPACE_ERR:"
runtimeWorkspaceBaseDir = "/config"
)
type runtimeWorkspaceExecutor interface {
exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error
}
type runtimeWorkspaceFileService struct {
auditRepo repository.WorkspaceFileAuditRepository
executor runtimeWorkspaceExecutor
}
type k8sRuntimeWorkspaceExecutor struct {
deploymentService *k8s.InstanceDeploymentService
}
type runtimeWorkspaceEntryPayload struct {
Name string `json:"name"`
Path string `json:"path"`
IsDir bool `json:"is_dir"`
Size int64 `json:"size"`
ModifiedAt time.Time `json:"modified_at"`
}
func NewRuntimeWorkspaceFileService(auditRepo repository.WorkspaceFileAuditRepository) WorkspaceFileService {
return &runtimeWorkspaceFileService{
auditRepo: auditRepo,
executor: &k8sRuntimeWorkspaceExecutor{
deploymentService: k8s.NewInstanceDeploymentService(),
},
}
}
func (s *runtimeWorkspaceFileService) List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
relative, err := cleanWorkspaceRelativePath(relativePath)
if err != nil {
return nil, err
}
var response struct {
Entries []runtimeWorkspaceEntryPayload `json:"entries"`
}
if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceListScript, runtimeWorkspaceBase(scope), relative}, nil, &response); err != nil {
return nil, err
}
entries := make([]WorkspaceEntry, 0, len(response.Entries))
for _, payload := range response.Entries {
entry := runtimeWorkspaceEntryFromPayload(payload)
entries = append(entries, entry)
}
sort.Slice(entries, func(i, j int) bool {
if entries[i].IsDir != entries[j].IsDir {
return entries[i].IsDir
}
left := strings.ToLower(entries[i].Name)
right := strings.ToLower(entries[j].Name)
if left == right {
return entries[i].Name < entries[j].Name
}
return left < right
})
return entries, nil
}
func (s *runtimeWorkspaceFileService) Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) {
relative, entry, err := s.statFile(ctx, scope, relativePath)
if err != nil {
return nil, err
}
kind, contentType, previewable, maxBytes := workspacePreviewKind(entry.Name, entry.Size)
downloadURL := workspaceDownloadURL(scope.InstanceID, relative)
if !previewable {
return &WorkspacePreview{
Kind: "binary",
ContentType: "application/octet-stream",
DownloadURL: downloadURL,
}, nil
}
if maxBytes > 0 && entry.Size > maxBytes {
return nil, ErrWorkspacePreviewTooLarge
}
if kind == "text" {
file, _, _, err := s.openRemoteFile(ctx, scope, relative, false)
if err != nil {
return nil, err
}
defer file.Close()
data, err := io.ReadAll(io.LimitReader(file, textPreviewMaxBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > textPreviewMaxBytes {
return nil, ErrWorkspacePreviewTooLarge
}
return &WorkspacePreview{
Kind: "text",
ContentType: contentType,
Text: string(data),
DownloadURL: downloadURL,
}, nil
}
return &WorkspacePreview{
Kind: kind,
ContentType: contentType,
PreviewURL: workspacePreviewURL(scope.InstanceID, relative),
DownloadURL: downloadURL,
}, nil
}
func (s *runtimeWorkspaceFileService) OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
_, entry, err := s.statFile(ctx, scope, relativePath)
if err != nil {
return nil, "", 0, err
}
_, contentType, previewable, maxBytes := workspacePreviewKind(entry.Name, entry.Size)
if !previewable {
return nil, "", 0, ErrWorkspaceFileExpected
}
if maxBytes > 0 && entry.Size > maxBytes {
return nil, "", 0, ErrWorkspacePreviewTooLarge
}
file, _, size, err := s.openRemoteFile(ctx, scope, relativePath, false)
if err != nil {
return nil, "", 0, err
}
return file, contentType, size, nil
}
func (s *runtimeWorkspaceFileService) OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
relative, entry, err := s.statFile(ctx, scope, relativePath)
if err != nil {
return nil, "", 0, err
}
file, filename, size, err := s.openRemoteFile(ctx, scope, relative, true)
if err != nil {
return nil, "", 0, err
}
if filename == "" {
filename = entry.Name
}
return file, filename, size, nil
}
func (s *runtimeWorkspaceFileService) Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
maxBytes := WorkspaceUploadMaxBytes()
if size > maxBytes {
return nil, ErrWorkspaceUploadTooLarge
}
cleanName, err := sanitizeWorkspaceFilename(filename)
if err != nil {
return nil, err
}
dir, err := cleanWorkspaceRelativePath(relativeDir)
if err != nil {
return nil, err
}
targetRelative := joinWorkspaceRelative(dir, cleanName)
limited := io.LimitReader(reader, maxBytes+1)
var payload runtimeWorkspaceEntryPayload
if err := s.runJSON(ctx, scope, []string{
"python3",
"-c",
runtimeWorkspaceUploadScript,
runtimeWorkspaceBase(scope),
dir,
cleanName,
fmt.Sprintf("%d", maxBytes),
}, limited, &payload); err != nil {
return nil, err
}
if err := s.recordAudit(ctx, scope, "upload", targetRelative, payload.Size); err != nil {
return nil, err
}
entry := runtimeWorkspaceEntryFromPayload(payload)
return &entry, nil
}
func (s *runtimeWorkspaceFileService) Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) {
if isWorkspaceRootRelative(relativePath) {
return nil, ErrWorkspaceRootOperation
}
relative, err := cleanWorkspaceRelativePath(relativePath)
if err != nil {
return nil, err
}
var payload runtimeWorkspaceEntryPayload
if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceMkdirScript, runtimeWorkspaceBase(scope), relative}, nil, &payload); err != nil {
return nil, err
}
if err := s.recordAudit(ctx, scope, "mkdir", relative, 0); err != nil {
return nil, err
}
entry := runtimeWorkspaceEntryFromPayload(payload)
return &entry, nil
}
func (s *runtimeWorkspaceFileService) Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) {
if isWorkspaceRootRelative(oldPath) || isWorkspaceRootRelative(newPath) {
return nil, ErrWorkspaceRootOperation
}
oldRelative, err := cleanWorkspaceRelativePath(oldPath)
if err != nil {
return nil, err
}
newRelative, err := cleanWorkspaceRelativePath(newPath)
if err != nil {
return nil, err
}
var payload runtimeWorkspaceEntryPayload
if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceRenameScript, runtimeWorkspaceBase(scope), oldRelative, newRelative}, nil, &payload); err != nil {
return nil, err
}
if err := s.recordAudit(ctx, scope, "rename", oldRelative+" -> "+newRelative, 0); err != nil {
return nil, err
}
entry := runtimeWorkspaceEntryFromPayload(payload)
return &entry, nil
}
func (s *runtimeWorkspaceFileService) Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error {
if isWorkspaceRootRelative(relativePath) {
return ErrWorkspaceRootOperation
}
relative, err := cleanWorkspaceRelativePath(relativePath)
if err != nil {
return err
}
if err := s.run(ctx, scope, []string{"python3", "-c", runtimeWorkspaceDeleteScript, runtimeWorkspaceBase(scope), relative}, nil, io.Discard); err != nil {
return err
}
return s.recordAudit(ctx, scope, "delete", relative, 0)
}
func (s *runtimeWorkspaceFileService) statFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (string, WorkspaceEntry, error) {
relative, err := cleanWorkspaceRelativePath(relativePath)
if err != nil {
return "", WorkspaceEntry{}, err
}
var payload runtimeWorkspaceEntryPayload
if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceStatFileScript, runtimeWorkspaceBase(scope), relative}, nil, &payload); err != nil {
return "", WorkspaceEntry{}, err
}
entry := runtimeWorkspaceEntryFromPayload(payload)
return relative, entry, nil
}
func (s *runtimeWorkspaceFileService) openRemoteFile(ctx context.Context, scope WorkspaceFileScope, relativePath string, audit bool) (*os.File, string, int64, error) {
relative, entry, err := s.statFile(ctx, scope, relativePath)
if err != nil {
return nil, "", 0, err
}
file, err := os.CreateTemp("", "clawmanager-runtime-workspace-*")
if err != nil {
return nil, "", 0, err
}
ok := false
defer func() {
if !ok {
name := file.Name()
_ = file.Close()
_ = os.Remove(name)
}
}()
if err := s.run(ctx, scope, []string{"python3", "-c", runtimeWorkspaceStreamFileScript, runtimeWorkspaceBase(scope), relative}, nil, file); err != nil {
return nil, "", 0, err
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return nil, "", 0, err
}
if audit {
if err := s.recordAudit(ctx, scope, "download", relative, entry.Size); err != nil {
return nil, "", 0, err
}
}
ok = true
return file, entry.Name, entry.Size, nil
}
func (s *runtimeWorkspaceFileService) runJSON(ctx context.Context, scope WorkspaceFileScope, command []string, stdin io.Reader, target any) error {
var stdout bytes.Buffer
if err := s.run(ctx, scope, command, stdin, &stdout); err != nil {
return err
}
if err := json.Unmarshal(stdout.Bytes(), target); err != nil {
return fmt.Errorf("failed to parse runtime workspace response: %w", err)
}
return nil
}
func (s *runtimeWorkspaceFileService) run(ctx context.Context, scope WorkspaceFileScope, command []string, stdin io.Reader, stdout io.Writer) error {
if s == nil || s.executor == nil {
return fmt.Errorf("runtime workspace executor is not configured")
}
var stderr bytes.Buffer
if err := s.executor.exec(ctx, scope.UserID, scope.InstanceID, command, stdin, stdout, &stderr); err != nil {
return mapRuntimeWorkspaceError(err, stderr.String())
}
return nil
}
func (s *runtimeWorkspaceFileService) recordAudit(ctx context.Context, scope WorkspaceFileScope, action, relativePath string, bytes int64) error {
if s.auditRepo == nil {
return nil
}
if err := ctx.Err(); err != nil {
return err
}
return s.auditRepo.Create(ctx, &models.WorkspaceFileAudit{
InstanceID: scope.InstanceID,
UserID: scope.UserID,
Action: action,
RelativePath: relativePath,
Bytes: bytes,
CreatedAt: time.Now().UTC(),
})
}
func (e *k8sRuntimeWorkspaceExecutor) exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error {
client := k8s.GetClient()
if client == nil || client.Clientset == nil || client.Config == nil {
return fmt.Errorf("k8s client not initialized")
}
deploymentService := e.deploymentService
if deploymentService == nil {
deploymentService = k8s.NewInstanceDeploymentService()
}
pod, err := deploymentService.GetActivePod(ctx, userID, instanceID)
if err != nil {
return fmt.Errorf("failed to get active instance pod: %w", err)
}
req := client.Clientset.CoreV1().RESTClient().Post().
Resource("pods").
Name(pod.Name).
Namespace(pod.Namespace).
SubResource("exec")
req.VersionedParams(&corev1.PodExecOptions{
Container: "desktop",
Command: command,
Stdin: stdin != nil,
Stdout: stdout != nil,
Stderr: stderr != nil,
TTY: false,
}, scheme.ParameterCodec)
exec, err := remotecommand.NewSPDYExecutor(client.Config, "POST", req.URL())
if err != nil {
return fmt.Errorf("failed to initialize runtime workspace exec stream: %w", err)
}
return exec.StreamWithContext(ctx, remotecommand.StreamOptions{
Stdin: stdin,
Stdout: stdout,
Stderr: stderr,
Tty: false,
})
}
func runtimeWorkspaceBase(scope WorkspaceFileScope) string {
if base := strings.TrimSpace(scope.WorkspacePath); base != "" {
return base
}
return runtimeWorkspaceBaseDir
}
func runtimeWorkspaceEntryFromPayload(payload runtimeWorkspaceEntryPayload) WorkspaceEntry {
isDir := payload.IsDir
return WorkspaceEntry{
Name: payload.Name,
Path: filepath.ToSlash(payload.Path),
IsDir: isDir,
Size: payload.Size,
ModifiedAt: payload.ModifiedAt.UTC(),
Previewable: workspaceEntryPreviewable(payload.Name, payload.Size, isDir),
Downloadable: !isDir,
}
}
func mapRuntimeWorkspaceError(execErr error, stderr string) error {
code := runtimeWorkspaceErrorCode(stderr)
switch code {
case "not_found":
return ErrWorkspacePathNotFound
case "escape":
return ErrWorkspacePathEscape
case "dir_required":
return ErrWorkspaceDirectoryExpected
case "file_required":
return ErrWorkspaceFileExpected
case "exists":
return ErrWorkspaceEntryExists
case "upload_too_large":
return ErrWorkspaceUploadTooLarge
case "filename_invalid", "invalid":
return ErrWorkspacePathInvalid
}
if strings.TrimSpace(stderr) != "" {
return fmt.Errorf("runtime workspace operation failed: %s", strings.TrimSpace(stderr))
}
if execErr != nil {
return execErr
}
return errors.New("runtime workspace operation failed")
}
func runtimeWorkspaceErrorCode(stderr string) string {
for _, line := range strings.Split(stderr, "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, runtimeWorkspaceErrPrefix) {
remainder := strings.TrimPrefix(line, runtimeWorkspaceErrPrefix)
if index := strings.Index(remainder, ":"); index >= 0 {
return remainder[:index]
}
return remainder
}
}
return ""
}
const runtimeWorkspacePythonPrelude = `
import datetime
import json
import os
import shutil
import stat
import sys
import tempfile
ERR_PREFIX = "CLAW_WORKSPACE_ERR:"
def fail(code, message=""):
print(ERR_PREFIX + code + ":" + str(message), file=sys.stderr)
sys.exit(1)
def clean_rel(value):
raw = (value or "").strip().replace("\\", "/")
if raw in ("", "."):
return ""
if "\x00" in raw:
fail("invalid", "path contains null byte")
if raw.startswith("/"):
fail("escape", "absolute paths are not allowed")
parts = []
for part in raw.split("/"):
if part in ("", "."):
continue
if part == "..":
fail("escape", "traversal is not allowed")
parts.append(part)
return "/".join(parts)
def safe_base(value):
base = os.path.realpath(value or "/config")
if not os.path.isdir(base):
fail("not_found", "workspace root")
return base
def is_subpath(base, target):
try:
return os.path.commonpath([base, target]) == base
except ValueError:
return False
def joined_path(base, rel):
target = os.path.abspath(os.path.join(base, rel)) if rel else base
if not is_subpath(base, target):
fail("escape", rel)
return target
def target_for(base, rel, allow_missing=False):
rel = clean_rel(rel)
target = joined_path(base, rel)
if os.path.lexists(target):
real = os.path.realpath(target)
if not is_subpath(base, real):
fail("escape", rel)
return rel, target
if allow_missing:
parent = os.path.realpath(os.path.dirname(target))
if not os.path.isdir(parent):
fail("invalid", "parent is not a directory")
if not is_subpath(base, parent):
fail("escape", rel)
return rel, target
fail("not_found", rel)
def entry_payload(base, rel, target):
try:
st = os.lstat(target)
except FileNotFoundError:
fail("not_found", rel)
name = os.path.basename(target) if rel else os.path.basename(base)
modified = datetime.datetime.fromtimestamp(st.st_mtime, datetime.timezone.utc).isoformat().replace("+00:00", "Z")
return {
"name": name,
"path": rel,
"is_dir": stat.S_ISDIR(st.st_mode),
"size": int(st.st_size),
"modified_at": modified,
}
def json_out(value):
print(json.dumps(value, separators=(",", ":")))
def chown_abc(path):
try:
import grp
import pwd
os.chown(path, pwd.getpwnam("abc").pw_uid, grp.getgrnam("abc").gr_gid)
except Exception:
pass
`
var runtimeWorkspaceListScript = runtimeWorkspacePythonPrelude + `
base = safe_base(sys.argv[1])
rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
if not os.path.isdir(target):
fail("dir_required", rel)
entries = []
for entry in os.scandir(target):
child_rel = (rel + "/" + entry.name) if rel else entry.name
child_path = os.path.join(target, entry.name)
try:
real = os.path.realpath(child_path)
if not is_subpath(base, real):
continue
except Exception:
continue
entries.append(entry_payload(base, child_rel, child_path))
json_out({"entries": entries})
`
var runtimeWorkspaceStatFileScript = runtimeWorkspacePythonPrelude + `
base = safe_base(sys.argv[1])
rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
if not os.path.isfile(target):
fail("file_required", rel)
json_out(entry_payload(base, rel, target))
`
var runtimeWorkspaceStreamFileScript = runtimeWorkspacePythonPrelude + `
base = safe_base(sys.argv[1])
rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
if not os.path.isfile(target):
fail("file_required", rel)
with open(target, "rb") as source:
shutil.copyfileobj(source, sys.stdout.buffer)
`
var runtimeWorkspaceUploadScript = runtimeWorkspacePythonPrelude + `
base = safe_base(sys.argv[1])
dir_rel, dir_path = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
name = sys.argv[3] if len(sys.argv) > 3 else ""
max_bytes = int(sys.argv[4]) if len(sys.argv) > 4 else 524288000
if not name or name in (".", "..") or "/" in name or "\\" in name or "\x00" in name:
fail("filename_invalid", name)
if not os.path.isdir(dir_path):
fail("dir_required", dir_rel)
target_rel = (dir_rel + "/" + name) if dir_rel else name
target_rel, target = target_for(base, target_rel, allow_missing=True)
if os.path.isdir(target):
fail("file_required", target_rel)
tmp_fd, tmp_name = tempfile.mkstemp(prefix="." + name + ".tmp-", dir=dir_path)
written = 0
try:
with os.fdopen(tmp_fd, "wb") as out:
while True:
chunk = sys.stdin.buffer.read(1024 * 1024)
if not chunk:
break
written += len(chunk)
if written > max_bytes:
fail("upload_too_large", target_rel)
out.write(chunk)
chown_abc(tmp_name)
os.replace(tmp_name, target)
chown_abc(target)
json_out(entry_payload(base, target_rel, target))
finally:
try:
if os.path.exists(tmp_name):
os.unlink(tmp_name)
except Exception:
pass
`
var runtimeWorkspaceMkdirScript = runtimeWorkspacePythonPrelude + `
base = safe_base(sys.argv[1])
rel = clean_rel(sys.argv[2] if len(sys.argv) > 2 else "")
if not rel:
fail("invalid", "workspace root cannot be modified")
rel, target = target_for(base, rel, allow_missing=True)
if os.path.lexists(target):
fail("exists", rel)
os.makedirs(target, mode=0o750, exist_ok=False)
chown_abc(target)
json_out(entry_payload(base, rel, target))
`
var runtimeWorkspaceRenameScript = runtimeWorkspacePythonPrelude + `
base = safe_base(sys.argv[1])
old_rel, old_target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
new_rel, new_target = target_for(base, sys.argv[3] if len(sys.argv) > 3 else "", allow_missing=True)
if not old_rel or not new_rel:
fail("invalid", "workspace root cannot be modified")
if os.path.lexists(new_target):
fail("exists", new_rel)
os.rename(old_target, new_target)
json_out(entry_payload(base, new_rel, new_target))
`
var runtimeWorkspaceDeleteScript = runtimeWorkspacePythonPrelude + `
base = safe_base(sys.argv[1])
rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
if not rel:
fail("invalid", "workspace root cannot be modified")
if os.path.islink(target) or os.path.isfile(target):
os.unlink(target)
elif os.path.isdir(target):
shutil.rmtree(target)
else:
fail("not_found", rel)
`
func runtimeWorkspaceContentType(name string) string {
contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(name)))
if contentType == "" {
return "application/octet-stream"
}
return contentType
}
backend/internal/services/sync_service.go
+99
View File
@@ -9,6 +9,7 @@ import (
"clawreef/internal/repository"
"clawreef/internal/services/k8s"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
)
@@ -17,6 +18,7 @@ type SyncService struct {
instanceRepo repository.InstanceRepository
runtimeStatusService InstanceRuntimeStatusService
podService *k8s.PodService
deploymentService *k8s.InstanceDeploymentService
interval time.Duration
stopChan chan struct{}
}
@@ -27,6 +29,7 @@ func NewSyncService(instanceRepo repository.InstanceRepository, runtimeStatusSer
instanceRepo: instanceRepo,
runtimeStatusService: runtimeStatusService,
podService: k8s.NewPodService(),
deploymentService: k8s.NewInstanceDeploymentService(),
interval: 5 * time.Second, // Sync every 5 seconds for more responsive status updates
stopChan: make(chan struct{}),
}
@@ -96,6 +99,15 @@ func (s *SyncService) syncAllInstances() {
// syncInstance synchronizes a single instance's state
func (s *SyncService) syncInstance(ctx context.Context, instance *models.Instance) {
if _, ok := v2RuntimeTypeForInstance(instance); ok {
s.updateInfraStatus(instance.ID, instance.Status)
return
}
if instanceUsesDesktopRuntime(instance) {
s.syncDeploymentInstance(ctx, instance)
return
}
// Check if pod exists in K8s
pod, err := s.podService.GetPod(ctx, instance.UserID, instance.ID)
if err != nil {
@@ -196,6 +208,73 @@ func (s *SyncService) syncInstance(ctx context.Context, instance *models.Instanc
}
}
func (s *SyncService) syncDeploymentInstance(ctx context.Context, instance *models.Instance) {
deployment, err := s.deploymentService.GetDeployment(ctx, instance.UserID, instance.ID)
if err != nil {
if instance.Status == "running" || instance.Status == "creating" {
nextStatus := "stopped"
if instance.Status == "creating" {
nextStatus = "error"
}
fmt.Printf("Instance %d marked as %s but deployment not found in K8s, updating status to %s\n",
instance.ID, instance.Status, nextStatus)
instance.Status = nextStatus
instance.PodName = nil
instance.PodNamespace = nil
instance.PodIP = nil
instance.UpdatedAt = time.Now()
if err := s.instanceRepo.Update(instance); err != nil {
fmt.Printf("Error updating instance %d status: %v\n", instance.ID, err)
} else {
s.updateInfraStatus(instance.ID, nextStatus)
GetHub().BroadcastInstanceStatus(instance.UserID, instance)
}
}
return
}
desiredStatus := mapDeploymentToInstanceStatus(deployment)
needsUpdate := false
if instance.Status != desiredStatus {
fmt.Printf("Instance %d: Deployment replicas=%d available=%d but instance status is %s, updating to %s\n",
instance.ID, desiredReplicas(deployment), deployment.Status.AvailableReplicas, instance.Status, desiredStatus)
instance.Status = desiredStatus
needsUpdate = true
}
s.updateInfraStatus(instance.ID, desiredStatus)
if pod, podErr := s.deploymentService.GetActivePod(ctx, instance.UserID, instance.ID); podErr == nil && pod != nil {
if pod.Status.PodIP != "" && (instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP) {
instance.PodIP = &pod.Status.PodIP
needsUpdate = true
}
if instance.PodName == nil || *instance.PodName != pod.Name {
instance.PodName = &pod.Name
needsUpdate = true
}
if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace {
instance.PodNamespace = &pod.Namespace
needsUpdate = true
}
} else if desiredStatus == "stopped" {
if instance.PodName != nil || instance.PodNamespace != nil || instance.PodIP != nil {
instance.PodName = nil
instance.PodNamespace = nil
instance.PodIP = nil
needsUpdate = true
}
}
if needsUpdate {
instance.UpdatedAt = time.Now()
if err := s.instanceRepo.Update(instance); err != nil {
fmt.Printf("Error updating instance %d: %v\n", instance.ID, err)
} else {
GetHub().BroadcastInstanceStatus(instance.UserID, instance)
}
}
}
func (s *SyncService) updateInfraStatus(instanceID int, instanceStatus string) {
if s.runtimeStatusService == nil {
return
@@ -243,6 +322,26 @@ func mapPodToInstanceStatus(pod *corev1.Pod) string {
}
}
func mapDeploymentToInstanceStatus(deployment *appsv1.Deployment) string {
if deployment == nil {
return "error"
}
if desiredReplicas(deployment) == 0 {
return "stopped"
}
if deployment.Status.AvailableReplicas > 0 {
return "running"
}
return "creating"
}
func desiredReplicas(deployment *appsv1.Deployment) int32 {
if deployment == nil || deployment.Spec.Replicas == nil {
return 1
}
return *deployment.Spec.Replicas
}
func isPodReady(pod *corev1.Pod) bool {
for _, condition := range pod.Status.Conditions {
if condition.Type == corev1.PodReady {
backend/internal/services/system_image_setting_service.go
+26 -16
View File
@@ -20,10 +20,10 @@ var orderedSystemImageTypes = []string{
}
var supportedSystemImageTypes = map[string]string{
"openclaw": "OpenClaw Desktop",
"openclaw": "OpenClaw Pro",
"ubuntu": "Ubuntu Desktop",
"webtop": "Webtop Desktop",
"hermes": "Hermes Runtime",
"hermes": "Hermes Pro",
"debian": "Debian Desktop",
"centos": "CentOS Desktop",
"custom": "Custom Image",
@@ -39,11 +39,11 @@ var defaultSystemImageSettings = map[string]string{
"custom": "registry.example.com/your-custom-image:latest",
}
var defaultShellSystemImageSettings = map[string]string{
"openclaw": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest",
var defaultGatewaySystemImageSettings = map[string]string{
"openclaw": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest",
"ubuntu": "ubuntu:22.04",
"webtop": "ubuntu:22.04",
"hermes": "ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest",
"hermes": "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest",
"debian": "debian:12",
"centos": "quay.io/centos/centos:stream9",
"custom": "registry.example.com/your-custom-shell-image:latest",
@@ -55,8 +55,9 @@ var defaultEnabledSystemImageTypes = map[string]bool{
"hermes": true,
}
var defaultEnabledShellSystemImageTypes = map[string]bool{
var defaultEnabledGatewaySystemImageTypes = map[string]bool{
"openclaw": true,
"hermes": true,
}
// RuntimeImageConfig is the runtime card selected for an instance type.
@@ -320,10 +321,16 @@ func displayNameForSystemImageType(instanceType string) string {
func displayNameForSystemImagePreset(instanceType, runtimeType string) string {
normalizedRuntimeType := normalizeSystemImageRuntimeType(runtimeType)
if instanceType == "openclaw" {
if normalizedRuntimeType == "shell" {
return "OpenClaw Shell"
if normalizedRuntimeType == "gateway" {
return "OpenClaw Lite"
}
return "OpenClaw Desktop"
return "OpenClaw Pro"
}
if instanceType == "hermes" {
if normalizedRuntimeType == "gateway" {
return "Hermes Lite"
}
return "Hermes Pro"
}
return displayNameForSystemImageType(instanceType)
}
@@ -337,12 +344,12 @@ func defaultSystemImagePresetsForType(instanceType string) []models.SystemImageS
IsEnabled: defaultEnabledSystemImageTypes[instanceType],
}}
if image := strings.TrimSpace(defaultShellSystemImageSettings[instanceType]); image != "" {
if defaultEnabledShellSystemImageTypes[instanceType] {
if image := strings.TrimSpace(defaultGatewaySystemImageSettings[instanceType]); image != "" {
if defaultEnabledGatewaySystemImageTypes[instanceType] {
settings = append(settings, models.SystemImageSetting{
InstanceType: instanceType,
RuntimeType: "shell",
DisplayName: displayNameForSystemImagePreset(instanceType, "shell"),
RuntimeType: "gateway",
DisplayName: displayNameForSystemImagePreset(instanceType, "gateway"),
Image: image,
IsEnabled: true,
})
@@ -359,8 +366,8 @@ func isSupportedSystemImageType(instanceType string) bool {
func normalizeSystemImageRuntimeType(runtimeType string) string {
normalized := strings.TrimSpace(strings.ToLower(runtimeType))
if normalized == "shell" {
return "shell"
if normalized == "gateway" || normalized == "shell" {
return "gateway"
}
return "desktop"
}
@@ -370,7 +377,10 @@ func validateSystemImageRuntimeType(runtimeType string) (string, error) {
if normalized == "" {
return "desktop", nil
}
if normalized != "desktop" && normalized != "shell" {
if normalized == "shell" {
return "gateway", nil
}
if normalized != "desktop" && normalized != "gateway" {
return "", errors.New("unsupported runtime type")
}
return normalized, nil
backend/internal/services/system_image_setting_service_test.go
+99 -20
View File
@@ -150,7 +150,7 @@ func TestSystemImageSettingServiceGetRuntimeImageFallsBackToDefaultWhenNoRowsExi
}
}
func TestSystemImageSettingServiceListIncludesOpenClawShellDefault(t *testing.T) {
func TestSystemImageSettingServiceListIncludesOpenClawGatewayDefault(t *testing.T) {
service := NewSystemImageSettingService(&stubSystemImageSettingRepository{})
items, err := service.List()
@@ -160,34 +160,63 @@ func TestSystemImageSettingServiceListIncludesOpenClawShellDefault(t *testing.T)
found := false
for _, item := range items {
if item.InstanceType == "openclaw" && item.RuntimeType == "shell" {
if item.InstanceType == "openclaw" && item.RuntimeType == "gateway" {
found = true
if item.Image != defaultShellSystemImageSettings["openclaw"] {
t.Fatalf("expected default openclaw shell image %q, got %q", defaultShellSystemImageSettings["openclaw"], item.Image)
if item.Image != defaultGatewaySystemImageSettings["openclaw"] {
t.Fatalf("expected default openclaw gateway image %q, got %q", defaultGatewaySystemImageSettings["openclaw"], item.Image)
}
if !item.IsEnabled {
t.Fatalf("expected default openclaw shell image to be enabled")
t.Fatalf("expected default openclaw gateway image to be enabled")
}
}
}
if !found {
t.Fatalf("expected default openclaw shell runtime image")
t.Fatalf("expected default openclaw gateway runtime image")
}
}
func TestSystemImageSettingServiceGetRuntimeImageForImageFallsBackToOpenClawShellDefault(t *testing.T) {
func TestSystemImageSettingServiceListIncludesHermesLiteDefault(t *testing.T) {
service := NewSystemImageSettingService(&stubSystemImageSettingRepository{})
selection, ok := service.GetRuntimeImageForImage("openclaw", defaultShellSystemImageSettings["openclaw"])
items, err := service.List()
if err != nil {
t.Fatalf("List returned error: %v", err)
}
found := false
for _, item := range items {
if item.InstanceType == "hermes" && item.RuntimeType == "gateway" {
found = true
if item.Image != defaultGatewaySystemImageSettings["hermes"] {
t.Fatalf("expected default hermes lite image %q, got %q", defaultGatewaySystemImageSettings["hermes"], item.Image)
}
if item.DisplayName != "Hermes Lite" {
t.Fatalf("expected Hermes Lite display name, got %q", item.DisplayName)
}
if !item.IsEnabled {
t.Fatalf("expected default hermes lite image to be enabled")
}
}
}
if !found {
t.Fatalf("expected default hermes lite runtime image")
}
}
func TestSystemImageSettingServiceGetRuntimeImageForImageFallsBackToOpenClawGatewayDefault(t *testing.T) {
service := NewSystemImageSettingService(&stubSystemImageSettingRepository{})
selection, ok := service.GetRuntimeImageForImage("openclaw", defaultGatewaySystemImageSettings["openclaw"])
if !ok {
t.Fatalf("expected default openclaw shell runtime image to resolve")
t.Fatalf("expected default openclaw gateway runtime image to resolve")
}
if selection.RuntimeType != "shell" {
t.Fatalf("expected runtime type shell, got %q", selection.RuntimeType)
if selection.RuntimeType != "gateway" {
t.Fatalf("expected runtime type gateway, got %q", selection.RuntimeType)
}
if selection.Image != defaultShellSystemImageSettings["openclaw"] {
t.Fatalf("expected shell image %q, got %q", defaultShellSystemImageSettings["openclaw"], selection.Image)
if selection.Image != defaultGatewaySystemImageSettings["openclaw"] {
t.Fatalf("expected gateway image %q, got %q", defaultGatewaySystemImageSettings["openclaw"], selection.Image)
}
}
@@ -195,20 +224,70 @@ func TestSystemImageSettingServiceGetRuntimeImageForImageUsesCardRuntimeType(t *
repo := &stubSystemImageSettingRepository{
items: []models.SystemImageSetting{
{ID: 1, InstanceType: "openclaw", RuntimeType: "desktop", DisplayName: "OpenClaw Desktop", Image: "registry/openclaw-desktop:latest", IsEnabled: true},
{ID: 2, InstanceType: "openclaw", RuntimeType: "shell", DisplayName: "OpenClaw Shell", Image: "registry/openclaw-shell:latest", IsEnabled: true},
{ID: 2, InstanceType: "openclaw", RuntimeType: "gateway", DisplayName: "OpenClaw Lite", Image: "registry/openclaw-lite:latest", IsEnabled: true},
},
nextID: 2,
}
service := NewSystemImageSettingService(repo)
selection, ok := service.GetRuntimeImageForImage("openclaw", "registry/openclaw-shell:latest")
selection, ok := service.GetRuntimeImageForImage("openclaw", "registry/openclaw-lite:latest")
if !ok {
t.Fatalf("expected selected shell runtime image to resolve")
t.Fatalf("expected selected gateway runtime image to resolve")
}
if selection.RuntimeType != "shell" {
t.Fatalf("expected runtime type shell, got %q", selection.RuntimeType)
if selection.RuntimeType != "gateway" {
t.Fatalf("expected runtime type gateway, got %q", selection.RuntimeType)
}
if selection.Image != "registry/openclaw-shell:latest" {
t.Fatalf("expected shell image to resolve, got %q", selection.Image)
if selection.Image != "registry/openclaw-lite:latest" {
t.Fatalf("expected gateway image to resolve, got %q", selection.Image)
}
}
func TestSystemImageSettingServiceSaveAcceptsGatewayRuntimeType(t *testing.T) {
repo := &stubSystemImageSettingRepository{}
service := NewSystemImageSettingService(repo)
saved, err := service.Save(&models.SystemImageSetting{
InstanceType: "OpenClaw",
RuntimeType: "gateway",
DisplayName: "OpenClaw Lite",
Image: "registry/openclaw-lite:v2",
})
if err != nil {
t.Fatalf("Save returned error: %v", err)
}
if saved.InstanceType != "openclaw" {
t.Fatalf("expected normalized instance type openclaw, got %q", saved.InstanceType)
}
if saved.RuntimeType != "gateway" {
t.Fatalf("expected gateway runtime type, got %q", saved.RuntimeType)
}
}
func TestSystemImageSettingServiceNormalizesLegacyShellRuntimeTypeToGateway(t *testing.T) {
repo := &stubSystemImageSettingRepository{
items: []models.SystemImageSetting{
{ID: 1, InstanceType: "hermes", RuntimeType: "shell", DisplayName: "Hermes Lite", Image: "registry/hermes-lite:legacy", IsEnabled: true},
},
nextID: 1,
}
service := NewSystemImageSettingService(repo)
items, err := service.List()
if err != nil {
t.Fatalf("List returned error: %v", err)
}
for _, item := range items {
if item.ID == 1 && item.RuntimeType != "gateway" {
t.Fatalf("expected legacy shell row to be exposed as gateway, got %q", item.RuntimeType)
}
}
selection, ok := service.GetRuntimeImageForImage("hermes", "registry/hermes-lite:legacy")
if !ok {
t.Fatalf("expected legacy shell image to resolve")
}
if selection.RuntimeType != "gateway" {
t.Fatalf("expected legacy shell image to resolve as gateway, got %q", selection.RuntimeType)
}
}
backend/internal/services/team_redis.go
+19
View File
@@ -70,6 +70,25 @@ func (b *redisBus) XAdd(ctx context.Context, key string, fields map[string]strin
return id, nil
}
func (b *redisBus) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) {
if ttl <= 0 {
ttl = time.Second
}
reply, err := b.do(ctx, "SET", key, value, "NX", "PX", fmt.Sprintf("%d", ttl.Milliseconds()))
if err != nil {
return false, err
}
if reply == nil {
return false, nil
}
return reply == "OK", nil
}
func (b *redisBus) Del(ctx context.Context, key string) error {
_, err := b.do(ctx, "DEL", key)
return err
}
func (b *redisBus) XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) {
blockMillis := int(block / time.Millisecond)
if blockMillis <= 0 {
backend/internal/services/team_service.go
+305 -85
View File
@@ -31,6 +31,8 @@ const (
teamTaskStaleSweepInterval = 30 * time.Second
initialLeaderTaskIntent = "team_bootstrap_introduction"
teamTaskCompletionTool = "team_complete_task"
teamTaskReplyTarget = "clawmanager"
)
var (
@@ -66,6 +68,8 @@ type CreateTeamMemberRequest struct {
MemberID string `json:"member_id,omitempty"`
Name string `json:"name,omitempty"`
Role string `json:"role"`
Mode string `json:"mode,omitempty"`
InstanceMode string `json:"instance_mode,omitempty"`
RuntimeType string `json:"runtime_type,omitempty"`
Description *string `json:"description,omitempty"`
CPUCores float64 `json:"cpu_cores,omitempty"`
@@ -130,36 +134,60 @@ type teamService struct {
secretService *k8s.SecretService
configMapService *k8s.ConfigMapService
ctx context.Context
cancel context.CancelFunc
mu sync.Mutex
consumers map[int]struct{}
staleMonitorStarted bool
ctx context.Context
cancel context.CancelFunc
mu sync.Mutex
consumers map[int]struct{}
staleMonitorStarted bool
runtimeWorkspaceRoot string
}
type plannedTeamMember struct {
Request CreateTeamMemberRequest
MemberKey string
DisplayName string
Role string
RuntimeType string
IsLeader bool
Request CreateTeamMemberRequest
MemberKey string
DisplayName string
Role string
RuntimeType string
InstanceMode string
IsLeader bool
}
func NewTeamService(repo repository.TeamRepository, instanceService InstanceService) TeamService {
ctx, cancel := context.WithCancel(context.Background())
return &teamService{
repo: repo,
instanceService: instanceService,
pvcService: k8s.NewPVCService(),
secretService: k8s.NewSecretService(),
configMapService: k8s.NewConfigMapService(),
ctx: ctx,
cancel: cancel,
consumers: map[int]struct{}{},
type teamRuntimeSecrets struct {
RedisURL string
Token string
}
type TeamServiceOption func(*teamService)
func WithTeamRuntimeWorkspaceRoot(root string) TeamServiceOption {
return func(s *teamService) {
if strings.TrimSpace(root) != "" {
s.runtimeWorkspaceRoot = strings.TrimSpace(root)
}
}
}
func NewTeamService(repo repository.TeamRepository, instanceService InstanceService, opts ...TeamServiceOption) TeamService {
ctx, cancel := context.WithCancel(context.Background())
service := &teamService{
repo: repo,
instanceService: instanceService,
pvcService: k8s.NewPVCService(),
secretService: k8s.NewSecretService(),
configMapService: k8s.NewConfigMapService(),
ctx: ctx,
cancel: cancel,
consumers: map[int]struct{}{},
runtimeWorkspaceRoot: "/workspaces",
}
for _, opt := range opts {
if opt != nil {
opt(service)
}
}
return service
}
func (s *teamService) Start() {
teams, err := s.repo.ListActiveTeams()
if err != nil {
@@ -252,15 +280,17 @@ func (s *teamService) CreateTeam(userID int, req CreateTeamRequest) (*TeamDetail
return nil, s.rollbackTeamCreation(userID, team, err)
}
if err := s.provisionTeamK8s(userID, team, redisURL, sharedStorageGB, strings.TrimSpace(req.StorageClass)); err != nil {
runtimeSecrets, err := s.provisionTeamK8s(userID, team, redisURL, sharedStorageGB, strings.TrimSpace(req.StorageClass))
if err != nil {
return nil, s.rollbackTeamCreation(userID, team, err)
}
if err := s.upsertTeamRosterConfig(userID, team, memberPlans); err != nil {
rosterJSON, err := s.upsertTeamRosterConfig(userID, team, memberPlans)
if err != nil {
return nil, s.rollbackTeamCreation(userID, team, err)
}
for _, memberPlan := range memberPlans {
member, err := s.createTeamMemberInstance(userID, team, memberPlan)
member, err := s.createTeamMemberInstance(userID, team, memberPlan, runtimeSecrets, rosterJSON)
if err != nil {
return nil, s.rollbackTeamCreation(userID, team, err)
}
@@ -339,16 +369,84 @@ func (s *teamService) recordInitialLeaderTaskDispatchFailure(teamID int, cause e
})
}
func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL string, sharedStorageGB int, storageClass string) error {
func buildTeamTaskEnvelope(teamID int, memberKey string, task *models.TeamTask, messageID string, taskPayload map[string]interface{}, now time.Time) map[string]interface{} {
if taskPayload == nil {
taskPayload = map[string]interface{}{}
}
taskID := 0
if task != nil {
taskID = task.ID
}
taskRef := fmt.Sprintf("team-%d-task-%d", teamID, taskID)
prompt := eventString(taskPayload, "prompt", "goal", "instruction", "instructions")
if prompt == "" {
rawPayload, _ := marshalJSON(taskPayload)
prompt = rawPayload
}
envelope := map[string]interface{}{
"v": 1,
"messageId": messageID,
"teamId": strconv.Itoa(teamID),
"from": "clawmanager",
"to": memberKey,
"replyTo": teamTaskReplyTarget,
"requiresCompletion": true,
"completionTool": teamTaskCompletionTool,
"resultSink": map[string]interface{}{
"type": "redis_stream",
"eventsKey": teamEventsKey(teamID),
"successEvent": "task_completed",
"failureEvent": "task_failed",
"replyEvent": "reply",
"resultField": "resultMarkdown",
"summaryField": "summary",
"artifactField": "artifactRefs",
"completionTool": teamTaskCompletionTool,
},
"intent": eventString(taskPayload, "intent"),
"taskId": taskRef,
"title": eventString(taskPayload, "title"),
"prompt": appendTeamTaskCompletionInstruction(prompt),
"contextRefs": normalizeContextRefs(taskPayload["contextRefs"]),
"metadata": taskPayload,
"createdAt": now.Format(time.RFC3339Nano),
}
if envelope["intent"] == "" {
envelope["intent"] = "run_task"
}
if envelope["title"] == "" {
envelope["title"] = fmt.Sprintf("Team task %d", taskID)
}
return envelope
}
func appendTeamTaskCompletionInstruction(prompt string) string {
base := strings.TrimSpace(prompt)
if strings.Contains(base, teamTaskCompletionTool) && strings.Contains(base, "task_completed") {
return base
}
instruction := strings.Join([]string{
"Completion contract:",
"- When the final result is ready, call team_complete_task with status=\"succeeded\", summary, and resultMarkdown.",
"- If the task fails, call team_complete_task with status=\"failed\" and an error message.",
"- Do not send the final answer as a normal message to clawmanager; ClawManager consumes task_completed/task_failed events from the Team Redis event stream.",
}, "\n")
if base == "" {
return instruction
}
return base + "\n\n" + instruction
}
func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL string, sharedStorageGB int, storageClass string) (*teamRuntimeSecrets, error) {
ctx := context.Background()
pvc, err := s.pvcService.CreateTeamSharedPVC(ctx, userID, team.ID, sharedStorageGB, storageClass)
if err != nil {
return err
return nil, err
}
secretName := s.pvcService.GetClient().GetTeamSecretName(team.ID)
teamToken, err := generatePrefixedToken("team")
if err != nil {
return fmt.Errorf("failed to generate Team token: %w", err)
return nil, fmt.Errorf("failed to generate Team token: %w", err)
}
if err := s.secretService.UpsertSecret(ctx, userID, secretName, map[string]string{
teamRedisURLSecretKey: redisURL,
@@ -358,7 +456,7 @@ func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL s
"managed-by": "clawreef",
"team-id": strconv.Itoa(team.ID),
}); err != nil {
return err
return nil, err
}
team.RedisURLSecretName = &secretName
@@ -368,21 +466,27 @@ func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL s
team.SharedPVCName = &pvc.Name
team.SharedPVCNamespace = &pvc.Namespace
team.UpdatedAt = time.Now().UTC()
return s.repo.UpdateTeam(team)
if err := s.repo.UpdateTeam(team); err != nil {
return nil, err
}
return &teamRuntimeSecrets{RedisURL: redisURL, Token: teamToken}, nil
}
func (s *teamService) upsertTeamRosterConfig(userID int, team *models.Team, members []plannedTeamMember) error {
func (s *teamService) upsertTeamRosterConfig(userID int, team *models.Team, members []plannedTeamMember) (string, error) {
rosterJSON, err := buildTeamRosterConfig(team, members)
if err != nil {
return err
return "", err
}
return s.configMapService.UpsertConfigMap(context.Background(), userID, s.teamConfigMapName(team.ID), map[string]string{
if err := s.configMapService.UpsertConfigMap(context.Background(), userID, s.teamConfigMapName(team.ID), map[string]string{
teamConfigFileName: rosterJSON,
}, map[string]string{
"app": "clawreef",
"managed-by": "clawreef",
"team-id": strconv.Itoa(team.ID),
})
}); err != nil {
return "", err
}
return rosterJSON, nil
}
func (s *teamService) teamConfigMapName(teamID int) string {
@@ -393,7 +497,7 @@ func (s *teamService) teamConfigMapName(teamID int) string {
return client.GetTeamConfigMapName(teamID)
}
func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, memberPlan plannedTeamMember) (*models.TeamMember, error) {
func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, memberPlan plannedTeamMember, runtimeSecrets *teamRuntimeSecrets, rosterJSON string) (*models.TeamMember, error) {
now := time.Now().UTC()
member := &models.TeamMember{
TeamID: team.ID,
@@ -402,6 +506,7 @@ func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, me
DisplayName: memberPlan.DisplayName,
Role: memberPlan.Role,
RuntimeType: memberPlan.RuntimeType,
InstanceMode: memberPlan.InstanceMode,
Description: optionalString(strings.TrimSpace(derefTeamString(memberPlan.Request.Description))),
Status: models.TeamMemberStatusCreating,
Availability: models.TeamMemberAvailabilityUnknown,
@@ -412,7 +517,7 @@ func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, me
return nil, err
}
createReq := s.buildTeamMemberInstanceRequest(team, memberPlan)
createReq := s.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, runtimeSecrets, rosterJSON)
instance, err := s.instanceService.Create(userID, createReq)
if err != nil {
member.Status = models.TeamMemberStatusFailed
@@ -437,10 +542,36 @@ func (s *teamService) buildTeamMemberInstanceRequests(team *models.Team, memberP
}
func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPlan plannedTeamMember) CreateInstanceRequest {
return s.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, nil, "")
}
func (s *teamService) buildTeamMemberInstanceRequestWithSecrets(team *models.Team, memberPlan plannedTeamMember, runtimeSecrets *teamRuntimeSecrets, rosterJSON string) CreateInstanceRequest {
req := memberPlan.Request
instanceMode := memberPlan.InstanceMode
if instanceMode == "" {
instanceMode = InstanceModeLite
}
runtimeBackendType, _ := RuntimeTypeForInstanceMode(instanceMode)
memberEnv := s.teamMemberEnv(team, memberPlan.MemberKey, memberPlan.Role)
if instanceMode == InstanceModeLite {
memberEnv["CLAWMANAGER_TEAM_SHARED_DIR"] = s.teamRuntimeSharedPath(team)
}
environmentOverrides := mergeEnvMaps(req.EnvironmentOverrides, memberEnv)
if instanceMode == InstanceModeLite && runtimeSecrets != nil {
environmentOverrides = mergeEnvMaps(environmentOverrides, map[string]string{
teamRedisURLSecretKey: runtimeSecrets.RedisURL,
teamTokenSecretKey: runtimeSecrets.Token,
})
if strings.TrimSpace(rosterJSON) != "" {
environmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"] = rosterJSONWithSharedDir(rosterJSON, s.teamRuntimeSharedPath(team))
}
}
return CreateInstanceRequest{
Name: teamMemberInstanceName(team.Name, team.ID, memberPlan.MemberKey),
Type: memberPlan.RuntimeType,
Mode: instanceMode,
InstanceMode: instanceMode,
RuntimeType: runtimeBackendType,
CPUCores: defaultFloat(req.CPUCores, 2),
MemoryGB: defaultInt(req.MemoryGB, 4),
DiskGB: defaultInt(req.DiskGB, 20),
@@ -450,7 +581,7 @@ func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPl
OSVersion: "latest",
ImageRegistry: req.ImageRegistry,
ImageTag: req.ImageTag,
EnvironmentOverrides: req.EnvironmentOverrides,
EnvironmentOverrides: environmentOverrides,
StorageClass: derefTeamString(team.StorageClass),
OpenClawConfigPlan: req.OpenClawConfigPlan,
Team: &TeamInstanceConfig{
@@ -467,6 +598,13 @@ func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPl
}
}
func (s *teamService) teamRuntimeSharedPath(team *models.Team) string {
if team == nil {
return k8s.TeamSharedWorkspacePath(s.runtimeWorkspaceRoot, 0, 0)
}
return k8s.TeamSharedWorkspacePath(s.runtimeWorkspaceRoot, team.UserID, team.ID)
}
func (s *teamService) teamMemberEnv(team *models.Team, memberKey, role string) map[string]string {
managerBaseURL, _ := defaultTeamManagerBaseURL()
return map[string]string{
@@ -651,30 +789,7 @@ func (s *teamService) DispatchTask(userID, teamID int, req DispatchTeamTaskReque
}
}
now := time.Now().UTC()
envelope := map[string]interface{}{
"v": 1,
"messageId": messageID,
"teamId": strconv.Itoa(teamID),
"from": "clawmanager",
"to": member.MemberKey,
"intent": eventString(taskPayload, "intent"),
"taskId": fmt.Sprintf("team-%d-task-%d", teamID, task.ID),
"title": eventString(taskPayload, "title"),
"prompt": eventString(taskPayload, "prompt", "goal", "instruction", "instructions"),
"contextRefs": normalizeContextRefs(taskPayload["contextRefs"]),
"metadata": taskPayload,
"createdAt": now.Format(time.RFC3339Nano),
}
if envelope["intent"] == "" {
envelope["intent"] = "run_task"
}
if envelope["title"] == "" {
envelope["title"] = fmt.Sprintf("Team task %d", task.ID)
}
if envelope["prompt"] == "" {
rawPayload, _ := marshalJSON(taskPayload)
envelope["prompt"] = rawPayload
}
envelope := buildTeamTaskEnvelope(teamID, member.MemberKey, task, messageID, taskPayload, now)
envelopeJSON, err := marshalJSON(envelope)
if err != nil {
return nil, fmt.Errorf("failed to encode task envelope: %w", err)
@@ -1090,6 +1205,7 @@ func (s *teamService) projectTeamEvent(team *models.Team, bus *redisBus, message
}
task = found
}
eventType = normalizeFinalReplyTaskEvent(eventType, payload, task)
payloadJSON, err := marshalOptionalJSON(payload)
if err != nil {
@@ -1184,6 +1300,35 @@ func (s *teamService) projectTeamEvent(team *models.Team, bus *redisBus, message
return nil
}
func normalizeFinalReplyTaskEvent(eventType string, payload map[string]interface{}, task *models.TeamTask) string {
if task == nil || !strings.EqualFold(strings.TrimSpace(eventType), "reply") {
return eventType
}
if !eventBool(payload, "final", "isFinal", "complete", "completed", "taskCompleted") {
return eventType
}
if !teamEventHasBody(payload) {
return eventType
}
payload["originalEvent"] = eventType
payload["event"] = "task_completed"
payload["type"] = "task_completed"
payload["status"] = "succeeded"
payload["availability"] = models.TeamMemberAvailabilityIdle
payload["runtimeStatus"] = "succeeded"
if eventString(payload, "resultMarkdown") == "" {
if text := eventString(payload, "text", "result", "summary"); text != "" {
payload["resultMarkdown"] = text
}
}
if eventString(payload, "summary") == "" {
if text := eventString(payload, "text", "resultMarkdown", "result"); text != "" {
payload["summary"] = text
}
}
return "task_completed"
}
func (s *teamService) enrichOutboundEventFromInbox(teamID int, bus *redisBus, payload map[string]interface{}, messageID string) (map[string]interface{}, error) {
targetMember := eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id")
if targetMember == "" {
@@ -1429,6 +1574,36 @@ func eventString(payload map[string]interface{}, keys ...string) string {
return ""
}
func eventBool(payload map[string]interface{}, keys ...string) bool {
for _, key := range keys {
value, ok := payload[key]
if !ok || value == nil {
continue
}
switch typed := value.(type) {
case bool:
return typed
case string:
switch strings.ToLower(strings.TrimSpace(typed)) {
case "1", "true", "yes", "y", "on":
return true
case "0", "false", "no", "n", "off":
return false
}
case float64:
return typed != 0
case int:
return typed != 0
default:
text := strings.ToLower(strings.TrimSpace(fmt.Sprintf("%v", typed)))
if text == "true" || text == "yes" || text == "1" {
return true
}
}
}
return false
}
func applyTeamMemberRuntimeProjection(member *models.TeamMember, payload map[string]interface{}, eventType string) {
if member == nil {
return
@@ -1557,6 +1732,10 @@ func planTeamMembers(teamName string, members []CreateTeamMemberRequest) ([]plan
if err != nil {
return nil, err
}
instanceMode, err := normalizeTeamMemberInstanceMode(memberReq.Mode, memberReq.InstanceMode)
if err != nil {
return nil, err
}
isLeader := memberReq.IsLeader || isTeamLeaderRole(role)
if isLeader {
@@ -1568,12 +1747,13 @@ func planTeamMembers(teamName string, members []CreateTeamMemberRequest) ([]plan
displayName = fmt.Sprintf("%s-%s", teamName, memberKey)
}
plans = append(plans, plannedTeamMember{
Request: memberReq,
MemberKey: memberKey,
DisplayName: displayName,
Role: role,
RuntimeType: runtimeType,
IsLeader: isLeader,
Request: memberReq,
MemberKey: memberKey,
DisplayName: displayName,
Role: role,
RuntimeType: runtimeType,
InstanceMode: instanceMode,
IsLeader: isLeader,
})
}
if leaderCount != 1 {
@@ -1636,6 +1816,22 @@ func normalizeTeamMemberRuntimeType(raw string) (string, error) {
}
}
func normalizeTeamMemberInstanceMode(rawMode, rawInstanceMode string) (string, error) {
if mode, ok := NormalizeInstanceMode(rawMode); ok {
return mode, nil
}
if strings.TrimSpace(rawMode) != "" {
return "", fmt.Errorf("unsupported team member instance mode: %s", rawMode)
}
if mode, ok := NormalizeInstanceMode(rawInstanceMode); ok {
return mode, nil
}
if strings.TrimSpace(rawInstanceMode) != "" {
return "", fmt.Errorf("unsupported team member instance mode: %s", rawInstanceMode)
}
return InstanceModeLite, nil
}
func normalizeTeamMemberRole(raw string, isLeader bool) string {
role := strings.TrimSpace(raw)
if isLeader || isTeamLeaderRole(role) {
@@ -1682,12 +1878,13 @@ type teamRosterConfig struct {
}
type teamRosterMember struct {
MemberID string `json:"memberId"`
Role string `json:"role"`
RuntimeType string `json:"runtimeType"`
DisplayName string `json:"displayName"`
Description string `json:"description,omitempty"`
IsLeader bool `json:"isLeader"`
MemberID string `json:"memberId"`
Role string `json:"role"`
RuntimeType string `json:"runtimeType"`
InstanceMode string `json:"instanceMode"`
DisplayName string `json:"displayName"`
Description string `json:"description,omitempty"`
IsLeader bool `json:"isLeader"`
}
type teamRosterRedis struct {
@@ -1714,12 +1911,13 @@ func buildTeamRosterConfig(team *models.Team, members []plannedTeamMember) (stri
config.LeaderMemberID = member.MemberKey
}
config.Members = append(config.Members, teamRosterMember{
MemberID: member.MemberKey,
Role: member.Role,
RuntimeType: member.RuntimeType,
DisplayName: member.DisplayName,
Description: derefTeamString(member.Request.Description),
IsLeader: member.IsLeader,
MemberID: member.MemberKey,
Role: member.Role,
RuntimeType: member.RuntimeType,
InstanceMode: member.InstanceMode,
DisplayName: member.DisplayName,
Description: derefTeamString(member.Request.Description),
IsLeader: member.IsLeader,
})
}
if config.LeaderMemberID == "" {
@@ -1728,6 +1926,23 @@ func buildTeamRosterConfig(team *models.Team, members []plannedTeamMember) (stri
return marshalJSON(config)
}
func rosterJSONWithSharedDir(rosterJSON, sharedDir string) string {
sharedDir = strings.TrimSpace(sharedDir)
if sharedDir == "" {
return rosterJSON
}
var payload map[string]interface{}
if err := json.Unmarshal([]byte(rosterJSON), &payload); err != nil {
return rosterJSON
}
payload["sharedDir"] = sharedDir
raw, err := json.Marshal(payload)
if err != nil {
return rosterJSON
}
return string(raw)
}
func buildTeamRosterConfigFromMembers(team *models.Team, members []models.TeamMember) (string, error) {
config := teamRosterConfig{
Version: 1,
@@ -1747,16 +1962,21 @@ func buildTeamRosterConfigFromMembers(team *models.Team, members []models.TeamMe
if runtimeType == "" {
runtimeType = "openclaw"
}
instanceMode := strings.TrimSpace(member.InstanceMode)
if instanceMode == "" {
instanceMode = InstanceModeLite
}
if isLeader {
config.LeaderMemberID = member.MemberKey
}
config.Members = append(config.Members, teamRosterMember{
MemberID: member.MemberKey,
Role: member.Role,
RuntimeType: runtimeType,
DisplayName: member.DisplayName,
Description: derefTeamString(member.Description),
IsLeader: isLeader,
MemberID: member.MemberKey,
Role: member.Role,
RuntimeType: runtimeType,
InstanceMode: instanceMode,
DisplayName: member.DisplayName,
Description: derefTeamString(member.Description),
IsLeader: isLeader,
})
}
if config.LeaderMemberID == "" {
backend/internal/services/team_service_test.go
+300 -1
View File
@@ -1,6 +1,7 @@
package services
import (
"encoding/json"
"strings"
"testing"
"time"
@@ -85,6 +86,81 @@ func TestBuildTeamMemberInstanceRequestUsesSharedPermissionDefaults(t *testing.T
}
}
func TestBuildTeamMemberInstanceRequestSupportsLiteMode(t *testing.T) {
service := &teamService{runtimeWorkspaceRoot: "/workspaces"}
team := &models.Team{
UserID: 1,
ID: 8,
Name: "Lite Team",
SharedMountPath: "/team",
}
memberPlan := plannedTeamMember{
Request: CreateTeamMemberRequest{
Mode: "Lite",
InstanceMode: "Lite",
},
MemberKey: "lite-worker",
DisplayName: "lite worker",
Role: "developer",
RuntimeType: "openclaw",
InstanceMode: InstanceModeLite,
}
req := service.buildTeamMemberInstanceRequest(team, memberPlan)
if req.Mode != InstanceModeLite || req.InstanceMode != InstanceModeLite {
t.Fatalf("expected Team member instance request to preserve lite mode, got mode=%q instance_mode=%q", req.Mode, req.InstanceMode)
}
if req.RuntimeType != RuntimeBackendGateway {
t.Fatalf("expected lite Team member to target gateway runtime, got %q", req.RuntimeType)
}
rosterJSON := `{"teamId":"8","members":[{"memberId":"lite-worker"}]}`
liteReq := service.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, &teamRuntimeSecrets{
RedisURL: "redis://team-redis:6379/0",
Token: "team_test_token",
}, rosterJSON)
if liteReq.EnvironmentOverrides[teamRedisURLSecretKey] != "redis://team-redis:6379/0" ||
liteReq.EnvironmentOverrides[teamTokenSecretKey] != "team_test_token" {
t.Fatalf("expected Lite Team runtime secrets in gateway env overrides, got %#v", liteReq.EnvironmentOverrides)
}
if !strings.Contains(liteReq.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"], `"sharedDir":"/workspaces/teams/user-1/team-8-shared"`) {
t.Fatalf("expected Lite Team roster JSON to include runtime shared dir, got %#v", liteReq.EnvironmentOverrides)
}
}
func TestBuildTeamMemberInstanceRequestPointsLiteSharedDirAtRuntimeWorkspace(t *testing.T) {
service := &teamService{runtimeWorkspaceRoot: "/workspaces"}
team := &models.Team{
UserID: 1,
ID: 28,
Name: "Mixed Team",
SharedMountPath: "/team",
}
memberPlan := plannedTeamMember{
MemberKey: "backend",
DisplayName: "backend",
Role: "developer",
RuntimeType: "openclaw",
InstanceMode: InstanceModeLite,
}
req := service.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, &teamRuntimeSecrets{
RedisURL: "redis://team-redis:6379/0",
Token: "team_test_token",
}, `{"sharedDir":"/team"}`)
wantSharedDir := "/workspaces/teams/user-1/team-28-shared"
if req.EnvironmentOverrides["CLAWMANAGER_TEAM_SHARED_DIR"] != wantSharedDir {
t.Fatalf("expected Lite shared dir %q, got %#v", wantSharedDir, req.EnvironmentOverrides)
}
if !strings.Contains(req.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"], `"sharedDir":"`+wantSharedDir+`"`) {
t.Fatalf("expected Lite roster JSON to use runtime shared dir, got %s", req.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"])
}
if req.Team.SharedMountPath != "/team" {
t.Fatalf("Pro Team mount path should remain /team, got %q", req.Team.SharedMountPath)
}
}
func TestNewRedisBusParsesURLWithoutNetwork(t *testing.T) {
bus, err := newRedisBus("redis://:pass@redis.example:6380/3")
if err != nil {
@@ -141,7 +217,7 @@ func TestPlanTeamMembersRequiresExactlyOneLeader(t *testing.T) {
func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) {
plans, err := planTeamMembers("team", []CreateTeamMemberRequest{
{MemberID: "lead", Role: "leader"},
{MemberID: "hermes-writer", Role: "writer", RuntimeType: "Hermes"},
{MemberID: "hermes-writer", Role: "writer", RuntimeType: "Hermes", InstanceMode: "Pro"},
})
if err != nil {
t.Fatalf("planTeamMembers returned error: %v", err)
@@ -149,6 +225,9 @@ func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) {
if plans[1].RuntimeType != "hermes" {
t.Fatalf("expected Hermes runtime to be normalized, got %#v", plans[1])
}
if plans[1].InstanceMode != InstanceModePro {
t.Fatalf("expected Pro instance mode to be normalized, got %#v", plans[1])
}
_, err = planTeamMembers("team", []CreateTeamMemberRequest{
{MemberID: "lead", Role: "leader"},
@@ -157,6 +236,14 @@ func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "unsupported team member runtime type") {
t.Fatalf("expected unsupported runtime validation error, got %v", err)
}
_, err = planTeamMembers("team", []CreateTeamMemberRequest{
{MemberID: "lead", Role: "leader"},
{MemberID: "worker", Role: "developer", Mode: "mini"},
})
if err == nil || !strings.Contains(err.Error(), "unsupported team member instance mode") {
t.Fatalf("expected unsupported instance mode validation error, got %v", err)
}
}
func TestTeamMemberInstanceNameUsesTeamIDAndMemberKey(t *testing.T) {
@@ -205,6 +292,9 @@ func TestBuildTeamRosterConfigOmitsSecrets(t *testing.T) {
if !strings.Contains(roster, `"runtimeType":"openclaw"`) {
t.Fatalf("roster missing member runtime type: %s", roster)
}
if !strings.Contains(roster, `"instanceMode":"lite"`) {
t.Fatalf("roster missing member instance mode: %s", roster)
}
}
func TestBuildInitialLeaderTaskPayloadDescribesRosterAndTeamSend(t *testing.T) {
@@ -236,6 +326,107 @@ func TestBuildInitialLeaderTaskPayloadDescribesRosterAndTeamSend(t *testing.T) {
}
}
func TestBuildTeamTaskEnvelopeIncludesCompletionContract(t *testing.T) {
task := &models.TeamTask{ID: 67}
payload := map[string]interface{}{
"intent": "team_bootstrap_introduction",
"title": "Introduce the team",
"prompt": "Generate the team report.",
}
envelope := buildTeamTaskEnvelope(31, "leader", task, "team-31-bootstrap-introduction", payload, time.Unix(123, 0).UTC())
if envelope["replyTo"] != "clawmanager" {
t.Fatalf("expected replyTo clawmanager, got %#v", envelope["replyTo"])
}
if envelope["requiresCompletion"] != true {
t.Fatalf("expected requiresCompletion=true, got %#v", envelope["requiresCompletion"])
}
if envelope["completionTool"] != "team_complete_task" {
t.Fatalf("expected completion tool team_complete_task, got %#v", envelope["completionTool"])
}
resultSink, ok := envelope["resultSink"].(map[string]interface{})
if !ok {
t.Fatalf("expected resultSink map, got %#v", envelope["resultSink"])
}
if resultSink["type"] != "redis_stream" || resultSink["eventsKey"] != "claw:team:31:events" {
t.Fatalf("unexpected resultSink: %#v", resultSink)
}
if resultSink["successEvent"] != "task_completed" || resultSink["failureEvent"] != "task_failed" {
t.Fatalf("unexpected resultSink events: %#v", resultSink)
}
prompt, ok := envelope["prompt"].(string)
if !ok {
t.Fatalf("expected prompt string, got %#v", envelope["prompt"])
}
for _, expected := range []string{"Generate the team report.", "team_complete_task", "resultMarkdown", "task_completed"} {
if !strings.Contains(prompt, expected) {
t.Fatalf("completion prompt missing %q: %s", expected, prompt)
}
}
}
func TestProjectTeamEventTreatsFinalReplyAsTaskCompleted(t *testing.T) {
taskID := 67
messageID := "team-31-bootstrap-introduction"
task := &models.TeamTask{
ID: taskID,
TeamID: 31,
TargetMemberID: 120,
MessageID: messageID,
Status: models.TeamTaskStatusRunning,
UpdatedAt: time.Now().UTC(),
}
member := &models.TeamMember{
ID: 120,
TeamID: 31,
MemberKey: "leader",
Status: models.TeamMemberStatusBusy,
CurrentTaskID: &taskID,
Availability: models.TeamMemberAvailabilityBusy,
}
repo := &teamRepositoryStub{
tasksByMessageID: map[string]*models.TeamTask{messageID: task},
membersByKey: map[string]*models.TeamMember{"leader": member},
}
service := &teamService{repo: repo}
payload := map[string]interface{}{
"event": "reply",
"messageId": messageID,
"memberId": "leader",
"taskId": "team-31-task-67",
"final": true,
"summary": "Team report ready",
"resultMarkdown": "Full report",
"text": "Full report",
}
payloadJSON, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{
ID: "1781171178655-0",
Fields: map[string]string{"payload": string(payloadJSON)},
})
if err != nil {
t.Fatalf("projectTeamEvent returned error: %v", err)
}
if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded {
t.Fatalf("expected final reply to mark task succeeded, got %#v", repo.updatedTask)
}
if repo.updatedTask.ResultJSON == nil || !strings.Contains(*repo.updatedTask.ResultJSON, "Full report") {
t.Fatalf("expected reply payload to become task result, got %#v", repo.updatedTask.ResultJSON)
}
if repo.updatedMember == nil || repo.updatedMember.Status != models.TeamMemberStatusIdle || repo.updatedMember.Availability != models.TeamMemberAvailabilityIdle {
t.Fatalf("expected member to become idle, got %#v", repo.updatedMember)
}
if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" {
t.Fatalf("expected stored event type task_completed, got %#v", repo.createdEvents)
}
}
func TestActiveTeamMembersFiltersDeletedMembers(t *testing.T) {
members := activeTeamMembers([]models.TeamMember{
{MemberKey: "leader", Status: models.TeamMemberStatusIdle},
@@ -343,3 +534,111 @@ func TestMergeMissingEventFieldsEnrichesOutboundPayload(t *testing.T) {
t.Fatalf("expected enriched event to have displayable body: %#v", merged)
}
}
type teamRepositoryStub struct {
teamsByID map[int]*models.Team
membersByID map[int]*models.TeamMember
membersByKey map[string]*models.TeamMember
tasksByID map[int]*models.TeamTask
tasksByMessageID map[string]*models.TeamTask
createdEvents []models.TeamEvent
updatedTask *models.TeamTask
updatedMember *models.TeamMember
updatedTeam *models.Team
}
func (s *teamRepositoryStub) CreateTeam(team *models.Team) error { return nil }
func (s *teamRepositoryStub) UpdateTeam(team *models.Team) error {
clone := *team
s.updatedTeam = &clone
return nil
}
func (s *teamRepositoryStub) GetTeamByID(id int) (*models.Team, error) {
if s.teamsByID != nil {
return s.teamsByID[id], nil
}
return nil, nil
}
func (s *teamRepositoryStub) GetTeamByUserIDAndName(userID int, name string) (*models.Team, error) {
return nil, nil
}
func (s *teamRepositoryStub) ExistsByUserIDAndName(userID int, name string) (bool, error) {
return false, nil
}
func (s *teamRepositoryStub) ListTeamsByUserID(userID int, offset, limit int) ([]models.Team, error) {
return nil, nil
}
func (s *teamRepositoryStub) ListActiveTeams() ([]models.Team, error) { return nil, nil }
func (s *teamRepositoryStub) CountTeamsByUserID(userID int) (int, error) {
return 0, nil
}
func (s *teamRepositoryStub) CreateMember(member *models.TeamMember) error { return nil }
func (s *teamRepositoryStub) UpdateMember(member *models.TeamMember) error {
clone := *member
s.updatedMember = &clone
return nil
}
func (s *teamRepositoryStub) GetMemberByID(id int) (*models.TeamMember, error) {
if s.membersByID != nil {
return s.membersByID[id], nil
}
return nil, nil
}
func (s *teamRepositoryStub) GetMemberByTeamKey(teamID int, memberKey string) (*models.TeamMember, error) {
if s.membersByKey == nil {
return nil, nil
}
member := s.membersByKey[memberKey]
if member == nil || member.TeamID != teamID {
return nil, nil
}
return member, nil
}
func (s *teamRepositoryStub) ListMembersByTeamID(teamID int) ([]models.TeamMember, error) {
return nil, nil
}
func (s *teamRepositoryStub) CreateTask(task *models.TeamTask) error { return nil }
func (s *teamRepositoryStub) UpdateTask(task *models.TeamTask) error {
clone := *task
s.updatedTask = &clone
return nil
}
func (s *teamRepositoryStub) GetTaskByID(id int) (*models.TeamTask, error) {
if s.tasksByID != nil {
return s.tasksByID[id], nil
}
return nil, nil
}
func (s *teamRepositoryStub) GetTaskByMessageID(teamID int, messageID string) (*models.TeamTask, error) {
if s.tasksByMessageID == nil {
return nil, nil
}
task := s.tasksByMessageID[messageID]
if task == nil || task.TeamID != teamID {
return nil, nil
}
return task, nil
}
func (s *teamRepositoryStub) ListTasksByTeamID(teamID int, limit int) ([]models.TeamTask, error) {
return nil, nil
}
func (s *teamRepositoryStub) ListTasksBeforeID(teamID, beforeID, limit int) ([]models.TeamTask, error) {
return nil, nil
}
func (s *teamRepositoryStub) ListStaleCandidateTasks(cutoff time.Time, limit int) ([]models.TeamTask, error) {
return nil, nil
}
func (s *teamRepositoryStub) CreateEvent(event *models.TeamEvent) error {
clone := *event
s.createdEvents = append(s.createdEvents, clone)
return nil
}
func (s *teamRepositoryStub) EventExistsByStreamID(teamID int, streamID string) (bool, error) {
return false, nil
}
func (s *teamRepositoryStub) ListEventsByTeamID(teamID int, limit int) ([]models.TeamEvent, error) {
return nil, nil
}
func (s *teamRepositoryStub) ListEventsBeforeID(teamID, beforeID, limit int) ([]models.TeamEvent, error) {
return nil, nil
}
backend/internal/services/websocket_service.go
+140 -1
View File
@@ -1,9 +1,11 @@
package services
import (
"context"
"encoding/json"
"log"
"net/http"
"strings"
"sync"
"time"
@@ -24,11 +26,20 @@ var upgrader = websocket.Upgrader{
type Client struct {
ID int
UserID int
Role string
Topic WebSocketTopic
Conn *websocket.Conn
Send chan []byte
hub *Hub
}
type WebSocketTopic string
const (
WebSocketTopicUser WebSocketTopic = "user"
WebSocketTopicRuntimeAdmin WebSocketTopic = "runtime_admin"
)
// Hub maintains the set of active clients and broadcasts messages
type Hub struct {
clients map[*Client]bool
@@ -181,16 +192,59 @@ func (h *Hub) BroadcastToAll(msgType string, data interface{}) {
h.broadcast <- msg
}
func (h *Hub) BroadcastRuntimeAdmin(msgType string, data interface{}) {
msg := &Message{
Type: msgType,
Data: data,
Timestamp: time.Now(),
}
h.broadcastRuntimeAdminMessage(msg)
}
func (h *Hub) broadcastRuntimeAdminMessage(msg *Message) {
h.mu.RLock()
clients := make([]*Client, 0, len(h.clients))
for client := range h.clients {
if client.Role == "admin" && client.Topic == WebSocketTopicRuntimeAdmin {
clients = append(clients, client)
}
}
h.mu.RUnlock()
encoded := mustEncode(msg)
for _, client := range clients {
select {
case client.Send <- encoded:
default:
h.mu.Lock()
if _, ok := h.clients[client]; ok {
close(client.Send)
delete(h.clients, client)
}
h.mu.Unlock()
}
}
}
// ServeWS handles WebSocket connections
func ServeWS(hub *Hub, w http.ResponseWriter, r *http.Request, userID int) {
ServeWSWithOptions(hub, w, r, userID, "", WebSocketTopicUser)
}
func ServeWSWithOptions(hub *Hub, w http.ResponseWriter, r *http.Request, userID int, role string, topic WebSocketTopic) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Printf("Failed to upgrade WebSocket: %v", err)
return
}
if topic == "" {
topic = WebSocketTopicUser
}
client := &Client{
UserID: userID,
Role: strings.TrimSpace(role),
Topic: topic,
Conn: conn,
Send: make(chan []byte, 256),
hub: hub,
@@ -203,6 +257,92 @@ func ServeWS(hub *Hub, w http.ResponseWriter, r *http.Request, userID int) {
go client.readPump()
}
func StartRuntimeAdminEventBridge(ctx context.Context, events RuntimeEventService, hub *Hub) {
if events == nil || hub == nil {
return
}
go func() {
lastID := "$"
for {
select {
case <-ctx.Done():
return
default:
}
messages, err := events.Read(ctx, lastID, 5*time.Second)
if err != nil {
if ctx.Err() != nil {
return
}
log.Printf("runtime admin event bridge read failed: %v", err)
sleepOrDone(ctx, time.Second)
continue
}
if len(messages) == 0 {
sleepOrDone(ctx, 200*time.Millisecond)
continue
}
for _, message := range messages {
if strings.TrimSpace(message.ID) != "" {
lastID = message.ID
}
if msg, ok := runtimeEventWebSocketMessage(message); ok {
hub.broadcastRuntimeAdminMessage(msg)
}
}
}
}()
}
func runtimeEventWebSocketMessage(message redisStreamMessage) (*Message, bool) {
eventType := strings.TrimSpace(message.Fields["type"])
if eventType == "" {
var event RuntimeEvent
if raw := strings.TrimSpace(message.Fields["event"]); raw != "" && json.Unmarshal([]byte(raw), &event) == nil {
eventType = strings.TrimSpace(event.Type)
if eventType == "" {
return nil, false
}
return &Message{
Type: eventType,
Data: json.RawMessage(event.Payload),
Timestamp: event.CreatedAt,
}, true
}
return nil, false
}
createdAt := time.Now().UTC()
if rawCreatedAt := strings.TrimSpace(message.Fields["created_at"]); rawCreatedAt != "" {
if parsed, err := time.Parse(time.RFC3339Nano, rawCreatedAt); err == nil {
createdAt = parsed
}
}
var data interface{} = map[string]any{}
if rawPayload := strings.TrimSpace(message.Fields["payload"]); rawPayload != "" {
if json.Valid([]byte(rawPayload)) {
data = json.RawMessage(rawPayload)
} else {
data = rawPayload
}
}
return &Message{
Type: eventType,
Data: data,
Timestamp: createdAt,
}, true
}
func sleepOrDone(ctx context.Context, d time.Duration) {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
// readPump pumps messages from the websocket connection to the hub
func (c *Client) readPump() {
defer func() {
@@ -278,7 +418,6 @@ func (h *Hub) GetClientCount() int {
return len(h.clients)
}
// Stop gracefully shuts down the hub, closing all client connections.
func (h *Hub) Stop() {
close(h.stop)
backend/internal/services/websocket_service_test.go
+55
View File
@@ -1,6 +1,7 @@
package services
import (
"encoding/json"
"sync"
"testing"
"time"
@@ -86,3 +87,57 @@ func TestHubStopClosesClients(t *testing.T) {
}
}
func TestHubBroadcastRuntimeAdminFiltersToAdminRuntimeClients(t *testing.T) {
h := NewHub()
adminRuntime := &Client{
UserID: 1,
Role: "admin",
Topic: WebSocketTopicRuntimeAdmin,
Send: make(chan []byte, 1),
hub: h,
}
adminUserTopic := &Client{
UserID: 2,
Role: "admin",
Topic: WebSocketTopicUser,
Send: make(chan []byte, 1),
hub: h,
}
normalRuntime := &Client{
UserID: 3,
Role: "user",
Topic: WebSocketTopicRuntimeAdmin,
Send: make(chan []byte, 1),
hub: h,
}
h.clients[adminRuntime] = true
h.clients[adminUserTopic] = true
h.clients[normalRuntime] = true
h.BroadcastRuntimeAdmin("runtime_pod_metrics", map[string]any{"pod_id": int64(9)})
select {
case raw := <-adminRuntime.Send:
var msg Message
if err := json.Unmarshal(raw, &msg); err != nil {
t.Fatalf("runtime admin message is not json: %v", err)
}
if msg.Type != "runtime_pod_metrics" {
t.Fatalf("message type = %q, want runtime_pod_metrics", msg.Type)
}
default:
t.Fatalf("admin runtime client did not receive runtime admin message")
}
select {
case raw := <-adminUserTopic.Send:
t.Fatalf("admin user-topic client received runtime admin message: %s", string(raw))
default:
}
select {
case raw := <-normalRuntime.Send:
t.Fatalf("normal runtime client received runtime admin message: %s", string(raw))
default:
}
}
backend/internal/services/workspace_file_service.go
+645
View File
@@ -0,0 +1,645 @@
package services
import (
"context"
"errors"
"fmt"
"io"
"mime"
"net/url"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"unicode"
"clawreef/internal/models"
"clawreef/internal/repository"
)
const (
WorkspaceUploadMaxBytesEnv = "WORKSPACE_UPLOAD_MAX_BYTES"
defaultWorkspaceUploadMax = int64(524288000)
textPreviewMaxBytes = int64(1024 * 1024)
imagePreviewMaxBytes = int64(10 * 1024 * 1024)
)
var (
ErrWorkspaceDirectoryExpected = errors.New("workspace directory is required")
ErrWorkspaceFileExpected = errors.New("workspace file is required")
ErrWorkspacePreviewTooLarge = errors.New("workspace preview exceeds maximum size")
ErrWorkspaceUploadTooLarge = errors.New("workspace upload exceeds maximum size")
ErrWorkspaceRootOperation = errors.New("workspace root cannot be modified")
ErrWorkspaceFileNameInvalid = errors.New("workspace filename is invalid")
ErrWorkspaceEntryExists = errors.New("workspace entry already exists")
)
type WorkspaceFileScope struct {
InstanceID int
UserID int
WorkspacePath string
}
type WorkspaceEntry struct {
Name string `json:"name"`
Path string `json:"path"`
IsDir bool `json:"is_dir"`
Size int64 `json:"size"`
ModifiedAt time.Time `json:"modified_at"`
Previewable bool `json:"previewable"`
Downloadable bool `json:"downloadable"`
}
type WorkspacePreview struct {
Kind string `json:"kind"`
ContentType string `json:"content_type"`
Text string `json:"text,omitempty"`
PreviewURL string `json:"preview_url,omitempty"`
DownloadURL string `json:"download_url,omitempty"`
}
type WorkspaceFileService interface {
List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error)
Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error)
OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error)
OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error)
Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error)
Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error)
Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error)
Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error
}
type workspaceFileService struct {
auditRepo repository.WorkspaceFileAuditRepository
}
func NewWorkspaceFileService(auditRepo repository.WorkspaceFileAuditRepository) WorkspaceFileService {
return &workspaceFileService{auditRepo: auditRepo}
}
func WorkspaceUploadMaxBytes() int64 {
raw := strings.TrimSpace(os.Getenv(WorkspaceUploadMaxBytesEnv))
if raw == "" {
return defaultWorkspaceUploadMax
}
value, err := strconv.ParseInt(raw, 10, 64)
if err != nil || value <= 0 {
return defaultWorkspaceUploadMax
}
return value
}
func (s *workspaceFileService) List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false)
if err != nil {
return nil, err
}
root, err := openWorkspaceRoot(scope.WorkspacePath)
if err != nil {
return nil, err
}
defer root.Close()
dir, err := root.Open(workspaceRootName(resolved.RelativePath))
if err != nil {
return nil, err
}
defer dir.Close()
info, err := dir.Stat()
if err != nil {
return nil, err
}
if !info.IsDir() {
return nil, ErrWorkspaceDirectoryExpected
}
items, err := dir.ReadDir(-1)
if err != nil {
return nil, err
}
entries := make([]WorkspaceEntry, 0, len(items))
for _, item := range items {
itemInfo, infoErr := item.Info()
if infoErr != nil {
return nil, infoErr
}
childRelative := joinWorkspaceRelative(resolved.RelativePath, item.Name())
entries = append(entries, buildWorkspaceEntry(childRelative, itemInfo))
}
sort.Slice(entries, func(i, j int) bool {
if entries[i].IsDir != entries[j].IsDir {
return entries[i].IsDir
}
left := strings.ToLower(entries[i].Name)
right := strings.ToLower(entries[j].Name)
if left == right {
return entries[i].Name < entries[j].Name
}
return left < right
})
return entries, nil
}
func (s *workspaceFileService) Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) {
resolved, file, info, err := s.openExistingFile(ctx, scope, relativePath)
if err != nil {
return nil, err
}
defer file.Close()
kind, contentType, previewable, maxBytes := workspacePreviewKind(info.Name(), info.Size())
downloadURL := workspaceDownloadURL(scope.InstanceID, resolved.RelativePath)
if !previewable {
return &WorkspacePreview{
Kind: "binary",
ContentType: "application/octet-stream",
DownloadURL: downloadURL,
}, nil
}
if maxBytes > 0 && info.Size() > maxBytes {
return nil, ErrWorkspacePreviewTooLarge
}
if kind == "text" {
data, err := io.ReadAll(io.LimitReader(file, textPreviewMaxBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > textPreviewMaxBytes {
return nil, ErrWorkspacePreviewTooLarge
}
return &WorkspacePreview{
Kind: "text",
ContentType: contentType,
Text: string(data),
DownloadURL: downloadURL,
}, nil
}
return &WorkspacePreview{
Kind: kind,
ContentType: contentType,
PreviewURL: workspacePreviewURL(scope.InstanceID, resolved.RelativePath),
DownloadURL: downloadURL,
}, nil
}
func (s *workspaceFileService) OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
file, info, contentType, err := s.openPreviewableFile(ctx, scope, relativePath)
if err != nil {
return nil, "", 0, err
}
return file, contentType, info.Size(), nil
}
func (s *workspaceFileService) openPreviewableFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, os.FileInfo, string, error) {
_, file, info, err := s.openExistingFile(ctx, scope, relativePath)
if err != nil {
return nil, nil, "", err
}
_, contentType, previewable, maxBytes := workspacePreviewKind(info.Name(), info.Size())
if !previewable {
file.Close()
return nil, nil, "", ErrWorkspaceFileExpected
}
if maxBytes > 0 && info.Size() > maxBytes {
file.Close()
return nil, nil, "", ErrWorkspacePreviewTooLarge
}
return file, info, contentType, nil
}
func (s *workspaceFileService) OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
resolved, file, info, err := s.openExistingFile(ctx, scope, relativePath)
if err != nil {
return nil, "", 0, err
}
if err := s.recordAudit(ctx, scope, "download", resolved.RelativePath, info.Size()); err != nil {
file.Close()
return nil, "", 0, err
}
return file, info.Name(), info.Size(), nil
}
func (s *workspaceFileService) Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
maxBytes := WorkspaceUploadMaxBytes()
if size > maxBytes {
return nil, ErrWorkspaceUploadTooLarge
}
cleanName, err := sanitizeWorkspaceFilename(filename)
if err != nil {
return nil, err
}
dir, err := ResolveWorkspacePath(scope.WorkspacePath, relativeDir, false)
if err != nil {
return nil, err
}
root, err := openWorkspaceRoot(scope.WorkspacePath)
if err != nil {
return nil, err
}
defer root.Close()
dirInfo, err := root.Stat(workspaceRootName(dir.RelativePath))
if err != nil {
return nil, err
}
if !dirInfo.IsDir() {
return nil, ErrWorkspaceDirectoryExpected
}
targetRelative := joinWorkspaceRelative(dir.RelativePath, cleanName)
target, err := ResolveWorkspacePath(scope.WorkspacePath, targetRelative, true)
if err != nil {
return nil, err
}
if err := ensureWorkspaceRuntimeOwnership(scope, dir.RelativePath); err != nil {
return nil, err
}
targetName := workspaceRootName(target.RelativePath)
if targetInfo, statErr := root.Stat(targetName); statErr == nil && targetInfo.IsDir() {
return nil, ErrWorkspaceFileExpected
} else if statErr != nil && !os.IsNotExist(statErr) {
return nil, statErr
}
tempRelative := joinWorkspaceRelative(dir.RelativePath, fmt.Sprintf(".%s.tmp-%d", cleanName, time.Now().UnixNano()))
tempName := workspaceRootName(tempRelative)
file, err := root.OpenFile(tempName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0640)
if err != nil {
return nil, err
}
written, copyErr := io.Copy(file, io.LimitReader(reader, maxBytes+1))
closeErr := file.Close()
if copyErr != nil {
_ = root.Remove(tempName)
return nil, copyErr
}
if closeErr != nil {
_ = root.Remove(tempName)
return nil, closeErr
}
if written > maxBytes {
_ = root.Remove(tempName)
return nil, ErrWorkspaceUploadTooLarge
}
if err := applyWorkspaceRuntimeOwnership(scope, tempRelative, 0640); err != nil {
_ = root.Remove(tempName)
return nil, err
}
if err := s.recordAudit(ctx, scope, "upload", target.RelativePath, written); err != nil {
_ = root.Remove(tempName)
return nil, err
}
if err := root.Rename(tempName, targetName); err != nil {
_ = root.Remove(tempName)
return nil, err
}
info, err := root.Stat(targetName)
if err != nil {
return nil, err
}
entry := buildWorkspaceEntry(target.RelativePath, info)
return &entry, nil
}
func (s *workspaceFileService) Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) {
if isWorkspaceRootRelative(relativePath) {
return nil, ErrWorkspaceRootOperation
}
resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, true)
if err != nil {
return nil, err
}
root, err := openWorkspaceRoot(scope.WorkspacePath)
if err != nil {
return nil, err
}
defer root.Close()
resolvedName := workspaceRootName(resolved.RelativePath)
if _, err := root.Lstat(resolvedName); err == nil {
return nil, ErrWorkspaceEntryExists
} else if !os.IsNotExist(err) {
return nil, err
}
if err := s.recordAudit(ctx, scope, "mkdir", resolved.RelativePath, 0); err != nil {
return nil, err
}
if err := root.MkdirAll(resolvedName, 0750); err != nil {
return nil, err
}
if err := ensureWorkspaceRuntimeOwnership(scope, resolved.RelativePath); err != nil {
_ = root.RemoveAll(resolvedName)
return nil, err
}
info, err := root.Stat(resolvedName)
if err != nil {
return nil, err
}
entry := buildWorkspaceEntry(resolved.RelativePath, info)
return &entry, nil
}
func (s *workspaceFileService) Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) {
if isWorkspaceRootRelative(oldPath) || isWorkspaceRootRelative(newPath) {
return nil, ErrWorkspaceRootOperation
}
oldResolved, err := ResolveWorkspacePath(scope.WorkspacePath, oldPath, false)
if err != nil {
return nil, err
}
newResolved, err := ResolveWorkspacePath(scope.WorkspacePath, newPath, true)
if err != nil {
return nil, err
}
root, err := openWorkspaceRoot(scope.WorkspacePath)
if err != nil {
return nil, err
}
defer root.Close()
oldName := workspaceRootName(oldResolved.RelativePath)
newName := workspaceRootName(newResolved.RelativePath)
if _, err := root.Lstat(newName); err == nil {
return nil, ErrWorkspaceEntryExists
} else if !os.IsNotExist(err) {
return nil, err
}
if _, err := root.Lstat(oldName); err != nil {
return nil, err
}
auditPath := oldResolved.RelativePath + " -> " + newResolved.RelativePath
if err := s.recordAudit(ctx, scope, "rename", auditPath, 0); err != nil {
return nil, err
}
if err := root.Rename(oldName, newName); err != nil {
return nil, err
}
if err := ensureWorkspaceRuntimeOwnership(scope, newResolved.RelativePath); err != nil {
return nil, err
}
info, err := root.Stat(newName)
if err != nil {
return nil, err
}
entry := buildWorkspaceEntry(newResolved.RelativePath, info)
return &entry, nil
}
func (s *workspaceFileService) Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error {
if isWorkspaceRootRelative(relativePath) {
return ErrWorkspaceRootOperation
}
resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false)
if err != nil {
return err
}
if err := s.recordAudit(ctx, scope, "delete", resolved.RelativePath, 0); err != nil {
return err
}
root, err := openWorkspaceRoot(scope.WorkspacePath)
if err != nil {
return err
}
defer root.Close()
return root.RemoveAll(workspaceRootName(resolved.RelativePath))
}
func (s *workspaceFileService) openExistingFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*ResolvedWorkspacePath, *os.File, os.FileInfo, error) {
if err := ctx.Err(); err != nil {
return nil, nil, nil, err
}
resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false)
if err != nil {
return nil, nil, nil, err
}
root, err := openWorkspaceRoot(scope.WorkspacePath)
if err != nil {
return nil, nil, nil, err
}
defer root.Close()
file, err := root.Open(workspaceRootName(resolved.RelativePath))
if err != nil {
return nil, nil, nil, err
}
info, err := file.Stat()
if err != nil {
file.Close()
return nil, nil, nil, err
}
if info.IsDir() {
file.Close()
return nil, nil, nil, ErrWorkspaceFileExpected
}
return resolved, file, info, nil
}
func (s *workspaceFileService) recordAudit(ctx context.Context, scope WorkspaceFileScope, action, relativePath string, bytes int64) error {
if s.auditRepo == nil {
return nil
}
if err := ctx.Err(); err != nil {
return err
}
return s.auditRepo.Create(ctx, &models.WorkspaceFileAudit{
InstanceID: scope.InstanceID,
UserID: scope.UserID,
Action: action,
RelativePath: relativePath,
Bytes: bytes,
CreatedAt: time.Now().UTC(),
})
}
func buildWorkspaceEntry(relativePath string, info os.FileInfo) WorkspaceEntry {
isDir := info.IsDir()
return WorkspaceEntry{
Name: info.Name(),
Path: filepath.ToSlash(relativePath),
IsDir: isDir,
Size: info.Size(),
ModifiedAt: info.ModTime().UTC(),
Previewable: workspaceEntryPreviewable(info.Name(), info.Size(), isDir),
Downloadable: !isDir,
}
}
func workspaceEntryPreviewable(name string, size int64, isDir bool) bool {
if isDir {
return false
}
kind, _, previewable, maxBytes := workspacePreviewKind(name, size)
if !previewable {
return false
}
return kind == "pdf" || maxBytes <= 0 || size <= maxBytes
}
func workspacePreviewKind(name string, size int64) (string, string, bool, int64) {
ext := strings.ToLower(filepath.Ext(name))
switch ext {
case ".txt", ".md", ".json", ".yaml", ".yml", ".log", ".py", ".js", ".ts", ".go", ".sh":
return "text", "text/plain; charset=utf-8", true, textPreviewMaxBytes
case ".png", ".jpg", ".jpeg", ".gif", ".webp":
contentType := mime.TypeByExtension(ext)
if contentType == "" {
contentType = "application/octet-stream"
}
return "image", contentType, true, imagePreviewMaxBytes
case ".pdf":
return "pdf", "application/pdf", true, 0
default:
return "binary", "application/octet-stream", false, 0
}
}
func sanitizeWorkspaceFilename(name string) (string, error) {
name = strings.TrimSpace(name)
if name == "" || name == "." || name == ".." {
return "", ErrWorkspaceFileNameInvalid
}
if strings.ContainsAny(name, `/\`) || strings.Contains(name, "\x00") {
return "", ErrWorkspaceFileNameInvalid
}
cleaned := strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return -1
}
switch r {
case ':', '*', '?', '"', '<', '>', '|':
return '-'
default:
return r
}
}, name)
cleaned = strings.TrimSpace(cleaned)
if cleaned == "" || cleaned == "." || cleaned == ".." {
return "", ErrWorkspaceFileNameInvalid
}
return cleaned, nil
}
func joinWorkspaceRelative(base, name string) string {
base = filepath.ToSlash(strings.TrimSpace(base))
name = filepath.ToSlash(strings.TrimSpace(name))
if base == "" {
return path.Clean(name)
}
return path.Clean(base + "/" + name)
}
func ensureWorkspaceRuntimeOwnership(scope WorkspaceFileScope, relativePath string) error {
if _, _, ok := runtimeWorkspaceOwner(scope); !ok {
return nil
}
relative, err := cleanWorkspaceRelativePath(relativePath)
if err != nil {
return err
}
if relative == "" {
return nil
}
parts := strings.Split(relative, "/")
for idx := range parts {
current := strings.Join(parts[:idx+1], "/")
resolved, err := ResolveWorkspacePath(scope.WorkspacePath, current, false)
if err != nil {
return err
}
info, err := os.Stat(resolved.RealPath)
if err != nil {
return err
}
mode := os.FileMode(0640)
if info.IsDir() {
mode = 0750
}
if err := applyWorkspaceRuntimeOwnershipToPath(scope, resolved.RealPath, mode); err != nil {
return err
}
}
return nil
}
func applyWorkspaceRuntimeOwnership(scope WorkspaceFileScope, relativePath string, mode os.FileMode) error {
if _, _, ok := runtimeWorkspaceOwner(scope); !ok {
return nil
}
resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false)
if err != nil {
return err
}
return applyWorkspaceRuntimeOwnershipToPath(scope, resolved.RealPath, mode)
}
func applyWorkspaceRuntimeOwnershipToPath(scope WorkspaceFileScope, targetPath string, mode os.FileMode) error {
uid, gid, ok := runtimeWorkspaceOwner(scope)
if !ok {
return nil
}
if err := os.Chown(targetPath, uid, gid); err != nil {
return err
}
if err := os.Chmod(targetPath, mode); err != nil {
return err
}
return nil
}
func runtimeWorkspaceOwner(scope WorkspaceFileScope) (int, int, bool) {
if scope.InstanceID <= 0 || scope.UserID <= 0 {
return 0, 0, false
}
cleaned := filepath.ToSlash(filepath.Clean(strings.TrimSpace(scope.WorkspacePath)))
parts := strings.Split(strings.Trim(cleaned, "/"), "/")
if len(parts) < 3 {
return 0, 0, false
}
instancePart := fmt.Sprintf("instance-%d", scope.InstanceID)
userPart := fmt.Sprintf("user-%d", scope.UserID)
runtimeType := parts[len(parts)-3]
if parts[len(parts)-1] != instancePart || parts[len(parts)-2] != userPart {
return 0, 0, false
}
if runtimeType != RuntimeTypeOpenClaw && runtimeType != RuntimeTypeHermes {
return 0, 0, false
}
linuxID := RuntimeLinuxID(scope.InstanceID)
return linuxID, linuxID, true
}
func isWorkspaceRootRelative(relativePath string) bool {
cleaned, err := cleanWorkspaceRelativePath(relativePath)
return err == nil && cleaned == ""
}
func openWorkspaceRoot(workspacePath string) (*os.Root, error) {
resolved, err := ResolveWorkspacePath(workspacePath, "", false)
if err != nil {
return nil, err
}
return os.OpenRoot(resolved.Root)
}
func workspaceRootName(relativePath string) string {
relativePath = filepath.ToSlash(strings.TrimSpace(relativePath))
if relativePath == "" {
return "."
}
return filepath.FromSlash(relativePath)
}
func workspaceDownloadURL(instanceID int, relativePath string) string {
return fmt.Sprintf("/api/v1/instances/%d/workspace/download?path=%s", instanceID, url.QueryEscape(filepath.ToSlash(relativePath)))
}
func workspacePreviewURL(instanceID int, relativePath string) string {
return fmt.Sprintf("/api/v1/instances/%d/workspace/preview?path=%s&raw=1", instanceID, url.QueryEscape(filepath.ToSlash(relativePath)))
}
backend/internal/services/workspace_file_service_test.go
+302
View File
@@ -0,0 +1,302 @@
package services
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"clawreef/internal/models"
)
type recordingWorkspaceFileAuditRepo struct {
items []models.WorkspaceFileAudit
err error
}
func (r *recordingWorkspaceFileAuditRepo) Create(ctx context.Context, audit *models.WorkspaceFileAudit) error {
if r.err != nil {
return r.err
}
r.items = append(r.items, *audit)
return nil
}
func workspaceTestScope(root string) WorkspaceFileScope {
return WorkspaceFileScope{
InstanceID: 12,
UserID: 34,
WorkspacePath: root,
}
}
func TestWorkspaceFileServiceListSortsDirectoriesFirstAndMarksCapabilities(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "docs"), 0750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "alpha.log"), []byte("hello"), 0640); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "binary.dat"), []byte{0, 1, 2}, 0640); err != nil {
t.Fatal(err)
}
service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
entries, err := service.List(context.Background(), workspaceTestScope(root), "")
if err != nil {
t.Fatalf("List returned error: %v", err)
}
names := make([]string, 0, len(entries))
for _, entry := range entries {
names = append(names, entry.Name)
}
if strings.Join(names, ",") != "docs,alpha.log,binary.dat" {
t.Fatalf("entry order = %v, want dirs first then names", names)
}
if entries[1].Path != "alpha.log" || !entries[1].Previewable || !entries[1].Downloadable {
t.Fatalf("alpha.log capabilities = %#v, want previewable downloadable file", entries[1])
}
if entries[2].Previewable || !entries[2].Downloadable {
t.Fatalf("binary.dat capabilities = %#v, want download-only file", entries[2])
}
}
func TestWorkspaceFileServicePreviewTextAndDoesNotAudit(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "notes.md"), []byte("# hello"), 0640); err != nil {
t.Fatal(err)
}
auditRepo := &recordingWorkspaceFileAuditRepo{}
service := NewWorkspaceFileService(auditRepo)
preview, err := service.Preview(context.Background(), workspaceTestScope(root), "notes.md")
if err != nil {
t.Fatalf("Preview returned error: %v", err)
}
if preview.Kind != "text" || preview.Text != "# hello" {
t.Fatalf("preview = %#v, want text preview", preview)
}
if len(auditRepo.items) != 0 {
t.Fatalf("preview audited %d events, want none", len(auditRepo.items))
}
}
func TestWorkspaceFileServiceTreatsSVGAsDownloadOnly(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "icon.svg"), []byte(`<svg onload="alert(1)"></svg>`), 0640); err != nil {
t.Fatal(err)
}
service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
entries, err := service.List(context.Background(), workspaceTestScope(root), "")
if err != nil {
t.Fatalf("List returned error: %v", err)
}
if len(entries) != 1 || entries[0].Previewable {
t.Fatalf("svg entry = %#v, want download-only", entries)
}
preview, err := service.Preview(context.Background(), workspaceTestScope(root), "icon.svg")
if err != nil {
t.Fatalf("Preview returned error: %v", err)
}
if preview.Kind != "binary" || preview.PreviewURL != "" {
t.Fatalf("svg preview = %#v, want binary without preview url", preview)
}
}
func TestWorkspaceFileServiceRejectsLargeTextPreview(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "large.log"), []byte(strings.Repeat("a", 1024*1024+1)), 0640); err != nil {
t.Fatal(err)
}
service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
_, err := service.Preview(context.Background(), workspaceTestScope(root), "large.log")
if !errors.Is(err, ErrWorkspacePreviewTooLarge) {
t.Fatalf("Preview error = %v, want ErrWorkspacePreviewTooLarge", err)
}
}
func TestWorkspaceFileServiceDownloadAuditsSuccessfulFileOpen(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "report.txt"), []byte("report"), 0640); err != nil {
t.Fatal(err)
}
auditRepo := &recordingWorkspaceFileAuditRepo{}
service := NewWorkspaceFileService(auditRepo)
file, filename, size, err := service.OpenDownload(context.Background(), workspaceTestScope(root), "report.txt")
if err != nil {
t.Fatalf("OpenDownload returned error: %v", err)
}
file.Close()
if filename != "report.txt" || size != int64(len("report")) {
t.Fatalf("download metadata filename=%q size=%d", filename, size)
}
if len(auditRepo.items) != 1 || auditRepo.items[0].Action != "download" || auditRepo.items[0].RelativePath != "report.txt" {
t.Fatalf("audit items = %#v, want download audit", auditRepo.items)
}
}
func TestWorkspaceFileServiceUploadRejectsEscapingSymlinkParent(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil {
t.Skipf("symlink creation is not available: %v", err)
}
service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
_, err := service.Upload(context.Background(), workspaceTestScope(root), "linked", "owned.txt", strings.NewReader("owned"), int64(len("owned")))
if !errors.Is(err, ErrWorkspacePathEscape) {
t.Fatalf("Upload error = %v, want ErrWorkspacePathEscape", err)
}
if _, statErr := os.Stat(filepath.Join(outside, "owned.txt")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("outside file stat error = %v, want not exist", statErr)
}
}
func TestWorkspaceFileServiceUploadEnforcesMaxBytesBeforeWriting(t *testing.T) {
t.Setenv(WorkspaceUploadMaxBytesEnv, "3")
root := t.TempDir()
service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
_, err := service.Upload(context.Background(), workspaceTestScope(root), "", "big.txt", strings.NewReader("abcd"), int64(len("abcd")))
if !errors.Is(err, ErrWorkspaceUploadTooLarge) {
t.Fatalf("Upload error = %v, want ErrWorkspaceUploadTooLarge", err)
}
if _, statErr := os.Stat(filepath.Join(root, "big.txt")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("uploaded file stat error = %v, want not exist", statErr)
}
}
func TestWorkspaceFileServiceRuntimeWorkspaceOwner(t *testing.T) {
scope := WorkspaceFileScope{
InstanceID: 74,
UserID: 1,
WorkspacePath: "/workspaces/openclaw/user-1/instance-74",
}
uid, gid, ok := runtimeWorkspaceOwner(scope)
if !ok {
t.Fatal("runtimeWorkspaceOwner ok = false, want true")
}
if uid != RuntimeLinuxID(74) || gid != RuntimeLinuxID(74) {
t.Fatalf("runtime owner uid:gid = %d:%d, want %d:%d", uid, gid, RuntimeLinuxID(74), RuntimeLinuxID(74))
}
scope.WorkspacePath = "/tmp/workspaces/openclaw/user-1/instance-74"
if _, _, ok := runtimeWorkspaceOwner(scope); !ok {
t.Fatal("runtimeWorkspaceOwner should support configurable workspace roots")
}
scope.WorkspacePath = "/workspaces/openclaw/user-2/instance-74"
if _, _, ok := runtimeWorkspaceOwner(scope); ok {
t.Fatal("runtimeWorkspaceOwner ok = true for mismatched user")
}
scope.WorkspacePath = "/var/lib/clawmanager/user-1/instance-74"
if _, _, ok := runtimeWorkspaceOwner(scope); ok {
t.Fatal("runtimeWorkspaceOwner ok = true for non-runtime workspace")
}
}
func TestWorkspaceFileServiceUploadAuditFailureRemovesFile(t *testing.T) {
root := t.TempDir()
auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")}
service := NewWorkspaceFileService(auditRepo)
_, err := service.Upload(context.Background(), workspaceTestScope(root), "", "created.txt", strings.NewReader("body"), int64(len("body")))
if err == nil || !strings.Contains(err.Error(), "audit unavailable") {
t.Fatalf("Upload error = %v, want audit failure", err)
}
if _, statErr := os.Stat(filepath.Join(root, "created.txt")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("created file stat error = %v, want not exist", statErr)
}
}
func TestWorkspaceFileServiceMkdirAuditFailureRemovesDirectory(t *testing.T) {
root := t.TempDir()
auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")}
service := NewWorkspaceFileService(auditRepo)
_, err := service.Mkdir(context.Background(), workspaceTestScope(root), "docs")
if err == nil || !strings.Contains(err.Error(), "audit unavailable") {
t.Fatalf("Mkdir error = %v, want audit failure", err)
}
if _, statErr := os.Stat(filepath.Join(root, "docs")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("created directory stat error = %v, want not exist", statErr)
}
}
func TestWorkspaceFileServiceRenameAuditFailureRollsBack(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "old.txt"), []byte("body"), 0640); err != nil {
t.Fatal(err)
}
auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")}
service := NewWorkspaceFileService(auditRepo)
_, err := service.Rename(context.Background(), workspaceTestScope(root), "old.txt", "new.txt")
if err == nil || !strings.Contains(err.Error(), "audit unavailable") {
t.Fatalf("Rename error = %v, want audit failure", err)
}
if _, statErr := os.Stat(filepath.Join(root, "old.txt")); statErr != nil {
t.Fatalf("old file stat error = %v, want rollback", statErr)
}
if _, statErr := os.Stat(filepath.Join(root, "new.txt")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("new file stat error = %v, want not exist", statErr)
}
}
func TestWorkspaceFileServiceDeleteAuditFailureKeepsFile(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "keep.txt"), []byte("body"), 0640); err != nil {
t.Fatal(err)
}
auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")}
service := NewWorkspaceFileService(auditRepo)
err := service.Delete(context.Background(), workspaceTestScope(root), "keep.txt")
if err == nil || !strings.Contains(err.Error(), "audit unavailable") {
t.Fatalf("Delete error = %v, want audit failure", err)
}
if _, statErr := os.Stat(filepath.Join(root, "keep.txt")); statErr != nil {
t.Fatalf("kept file stat error = %v", statErr)
}
}
func TestWorkspaceFileServiceMutationsAuditAndProtectRoot(t *testing.T) {
root := t.TempDir()
auditRepo := &recordingWorkspaceFileAuditRepo{}
service := NewWorkspaceFileService(auditRepo)
scope := workspaceTestScope(root)
if err := service.Delete(context.Background(), scope, ""); !errors.Is(err, ErrWorkspaceRootOperation) {
t.Fatalf("Delete root error = %v, want ErrWorkspaceRootOperation", err)
}
if _, err := service.Rename(context.Background(), scope, "", "renamed"); !errors.Is(err, ErrWorkspaceRootOperation) {
t.Fatalf("Rename root error = %v, want ErrWorkspaceRootOperation", err)
}
if _, err := service.Mkdir(context.Background(), scope, "docs"); err != nil {
t.Fatalf("Mkdir returned error: %v", err)
}
if _, err := service.Rename(context.Background(), scope, "docs", "notes"); err != nil {
t.Fatalf("Rename returned error: %v", err)
}
if err := service.Delete(context.Background(), scope, "notes"); err != nil {
t.Fatalf("Delete returned error: %v", err)
}
actions := make([]string, 0, len(auditRepo.items))
for _, item := range auditRepo.items {
actions = append(actions, item.Action)
}
if strings.Join(actions, ",") != "mkdir,rename,delete" {
t.Fatalf("audit actions = %v, want mkdir,rename,delete", actions)
}
}
backend/internal/services/workspace_path_guard.go
+202
View File
@@ -0,0 +1,202 @@
package services
import (
"errors"
"fmt"
"os"
"path"
"path/filepath"
"runtime"
"strings"
)
var (
ErrWorkspacePathInvalid = errors.New("workspace path is invalid")
ErrWorkspacePathEscape = errors.New("workspace path escapes workspace")
ErrWorkspacePathNotFound = errors.New("workspace entry not found")
)
type ResolvedWorkspacePath struct {
Root string
Path string
RealPath string
RelativePath string
}
func ResolveWorkspacePath(workspaceRoot, relativePath string, allowMissingLeaf bool) (*ResolvedWorkspacePath, error) {
root := strings.TrimSpace(workspaceRoot)
if root == "" {
return nil, fmt.Errorf("%w: workspace root is required", ErrWorkspacePathInvalid)
}
if strings.Contains(root, "\x00") {
return nil, fmt.Errorf("%w: workspace root contains null byte", ErrWorkspacePathInvalid)
}
rootAbs, err := filepath.Abs(root)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err)
}
rootReal, err := filepath.EvalSymlinks(rootAbs)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("%w: workspace root", ErrWorkspacePathNotFound)
}
return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err)
}
rootReal = filepath.Clean(rootReal)
rootInfo, err := os.Stat(rootReal)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("%w: workspace root", ErrWorkspacePathNotFound)
}
return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err)
}
if !rootInfo.IsDir() {
return nil, fmt.Errorf("%w: workspace root is not a directory", ErrWorkspacePathInvalid)
}
relative, err := cleanWorkspaceRelativePath(relativePath)
if err != nil {
return nil, err
}
targetPath := rootReal
if relative != "" {
targetPath = filepath.Join(rootReal, filepath.FromSlash(relative))
}
targetPath = filepath.Clean(targetPath)
if !isWorkspaceSubpath(rootReal, targetPath) {
return nil, fmt.Errorf("%w: %s", ErrWorkspacePathEscape, relativePath)
}
realPath, err := resolveWorkspaceRealPath(targetPath, allowMissingLeaf)
if err != nil {
return nil, err
}
realPath = filepath.Clean(realPath)
if !isWorkspaceSubpath(rootReal, realPath) {
return nil, fmt.Errorf("%w: %s", ErrWorkspacePathEscape, relativePath)
}
return &ResolvedWorkspacePath{
Root: rootReal,
Path: targetPath,
RealPath: realPath,
RelativePath: relative,
}, nil
}
func cleanWorkspaceRelativePath(relativePath string) (string, error) {
raw := strings.TrimSpace(relativePath)
if raw == "" || raw == "." {
return "", nil
}
if strings.Contains(raw, "\x00") {
return "", fmt.Errorf("%w: path contains null byte", ErrWorkspacePathInvalid)
}
if filepath.IsAbs(raw) || filepath.VolumeName(raw) != "" {
return "", fmt.Errorf("%w: absolute paths are not allowed", ErrWorkspacePathEscape)
}
normalized := strings.ReplaceAll(raw, "\\", "/")
if path.IsAbs(normalized) || isWindowsDrivePath(normalized) {
return "", fmt.Errorf("%w: absolute paths are not allowed", ErrWorkspacePathEscape)
}
parts := strings.Split(normalized, "/")
cleaned := make([]string, 0, len(parts))
for _, part := range parts {
switch part {
case "", ".":
continue
case "..":
return "", fmt.Errorf("%w: traversal is not allowed", ErrWorkspacePathEscape)
default:
cleaned = append(cleaned, part)
}
}
if len(cleaned) == 0 {
return "", nil
}
return path.Clean(strings.Join(cleaned, "/")), nil
}
func isWindowsDrivePath(value string) bool {
if len(value) < 2 || value[1] != ':' {
return false
}
first := value[0]
return (first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z')
}
func resolveWorkspaceRealPath(targetPath string, allowMissingLeaf bool) (string, error) {
realPath, err := filepath.EvalSymlinks(targetPath)
if err == nil {
return realPath, nil
}
if !os.IsNotExist(err) {
return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err)
}
if !allowMissingLeaf {
return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, targetPath)
}
existingPath := targetPath
missingParts := []string{}
for {
info, statErr := os.Lstat(existingPath)
if statErr == nil {
if len(missingParts) > 0 {
info, statErr = os.Stat(existingPath)
if statErr != nil {
if os.IsNotExist(statErr) {
return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, existingPath)
}
return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, statErr)
}
if !info.IsDir() {
return "", fmt.Errorf("%w: parent is not a directory", ErrWorkspacePathInvalid)
}
}
parentReal, evalErr := filepath.EvalSymlinks(existingPath)
if evalErr != nil {
if os.IsNotExist(evalErr) {
return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, existingPath)
}
return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, evalErr)
}
parts := append([]string{parentReal}, missingParts...)
return filepath.Join(parts...), nil
}
if !os.IsNotExist(statErr) {
return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, statErr)
}
parent := filepath.Dir(existingPath)
if parent == existingPath {
return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, targetPath)
}
missingParts = append([]string{filepath.Base(existingPath)}, missingParts...)
existingPath = parent
}
}
func isWorkspaceSubpath(root, target string) bool {
root = filepath.Clean(root)
target = filepath.Clean(target)
if sameWorkspacePath(root, target) {
return true
}
relative, err := filepath.Rel(root, target)
if err != nil || relative == "." || filepath.IsAbs(relative) {
return false
}
return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
}
func sameWorkspacePath(left, right string) bool {
if runtime.GOOS == "windows" {
return strings.EqualFold(left, right)
}
return left == right
}
backend/internal/services/workspace_path_guard_test.go
+76
View File
@@ -0,0 +1,76 @@
package services
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveWorkspacePathRejectsTraversalAndAbsolutePaths(t *testing.T) {
root := t.TempDir()
for _, relativePath := range []string{
"../secret.txt",
"docs/../../secret.txt",
"/etc/passwd",
`C:\Windows\System32\drivers\etc\hosts`,
"C:/Windows/System32/drivers/etc/hosts",
} {
t.Run(relativePath, func(t *testing.T) {
_, err := ResolveWorkspacePath(root, relativePath, false)
if !errors.Is(err, ErrWorkspacePathEscape) && !errors.Is(err, ErrWorkspacePathInvalid) {
t.Fatalf("ResolveWorkspacePath(%q) error = %v, want path safety error", relativePath, err)
}
})
}
}
func TestResolveWorkspacePathAllowsMissingChildInsideWorkspace(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "docs"), 0750); err != nil {
t.Fatal(err)
}
resolved, err := ResolveWorkspacePath(root, "docs/new.txt", true)
if err != nil {
t.Fatalf("ResolveWorkspacePath returned error: %v", err)
}
if resolved.RelativePath != "docs/new.txt" {
t.Fatalf("relative path = %q, want docs/new.txt", resolved.RelativePath)
}
if !strings.HasSuffix(filepath.ToSlash(resolved.Path), "/docs/new.txt") {
t.Fatalf("target path = %q, want path under docs/new.txt", resolved.Path)
}
}
func TestResolveWorkspacePathRejectsExistingSymlinkEscape(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("secret"), 0640); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join(outside, "secret.txt"), filepath.Join(root, "leak.txt")); err != nil {
t.Skipf("symlink creation is not available: %v", err)
}
_, err := ResolveWorkspacePath(root, "leak.txt", false)
if !errors.Is(err, ErrWorkspacePathEscape) {
t.Fatalf("ResolveWorkspacePath through symlink error = %v, want ErrWorkspacePathEscape", err)
}
}
func TestResolveWorkspacePathRejectsMissingTargetUnderEscapingSymlinkParent(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil {
t.Skipf("symlink creation is not available: %v", err)
}
_, err := ResolveWorkspacePath(root, "linked/new.txt", true)
if !errors.Is(err, ErrWorkspacePathEscape) {
t.Fatalf("ResolveWorkspacePath through symlink parent error = %v, want ErrWorkspacePathEscape", err)
}
}

Some files were not shown because too many files have changed in this diff Show More