diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e64f70a..fd84272 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -166,6 +166,12 @@ jobs: text, ) + text = re.sub( + r"(\n\s+- name: workspaces\s+)persistentVolumeClaim:\s+claimName: clawmanager-workspaces", + r"\1emptyDir: {}", + text, + ) + path.write_text(text, encoding="utf-8") PY diff --git a/.gitignore b/.gitignore index 0db0e93..2ff3ea2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.dylib /backend/bin/ /backend/tmp/ +/runtimeagent/ # Test binary, built with `go test -c` *.test @@ -61,6 +62,14 @@ Thumbs.db /fix-install-plan.md /docs/deployment-storage-remediation-plan.zh-CN.md +# Local API / debug artifacts (do not commit) +/_*.json +/_*.txt +/_test_*.js +/_pr*.md +.clawmanager-last-image +/pr138.patch + # Team runtime config snapshots /team-*.json /agency-agents-main/ diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 919f540..a720fff 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -1,4 +1,4 @@ -package main +package main import ( "context" @@ -104,7 +104,7 @@ func main() { log.Fatalf("Failed to initialize object storage: %v", err) } skillScannerClient := services.NewSkillScannerClient(cfg.SkillScanner) - aiObservabilityService := services.NewAIObservabilityService(modelInvocationRepo, auditEventRepo, costRecordRepo, riskHitRepo, chatMessageRepo, llmModelRepo, instanceRepo, userRepo) + aiObservabilityService := services.NewAIObservabilityService(modelInvocationRepo, auditEventRepo, costRecordRepo, riskHitRepo, chatMessageRepo, chatSessionRepo, llmModelRepo, instanceRepo, userRepo, instanceRuntimeStatusRepo) clusterResourceService := services.NewClusterResourceService(instanceRepo) services.SetRuntimeImageSettingsProvider(systemImageSettingService) services.SetOpenClawTransferRuntimeRepositories(instanceRepo, bindingRepo, runtimePodRepo) @@ -140,7 +140,19 @@ func main() { runtimeEvents := services.NewRuntimeEventService(platformRedis) workspaceFileService := services.NewWorkspaceFileService(workspaceFileAuditRepo) runtimeWorkspaceFileService := services.NewRuntimeWorkspaceFileService(workspaceFileAuditRepo) - skillService := services.NewSkillService(skillRepo, instanceRepo, instanceCommandService, objectStorageService, skillScannerClient) + skillService := services.NewSkillService(skillRepo, instanceRepo, userRepo, instanceCommandService, instanceCommandRepo, objectStorageService, skillScannerClient) + materializeJobRepo := repository.NewSkillPackageMaterializeJobRepository(database) + materializeService := services.NewSkillPackageMaterializeService(materializeJobRepo, skillRepo, services.SkillServiceAsMaterializer(skillService)) + services.ConfigureSkillPackageMaterialize(skillService, materializeService) + materializeWorker := services.NewSkillPackageMaterializeWorker( + materializeService, + time.Duration(cfg.SkillMaterialize.TickMS)*time.Millisecond, + cfg.SkillMaterialize.BatchSize, + cfg.SkillMaterialize.Concurrency, + cfg.SkillMaterialize.PerInstanceConcurrency, + cfg.SkillMaterialize.Enabled, + ) + services.ConfigureSkillRuntimeSync(skillService, bindingRepo, runtimePodRepo, runtimeAgentClient) securityScanService := services.NewSecurityScanService(securityScanRepo, skillRepo, objectStorageService, skillScannerClient) externalAccessService := services.NewInstanceExternalAccessService(instanceExternalAccessRepo) aiGatewayService := aigateway.NewService(llmModelRepo, modelInvocationService, auditEventService, costRecordService, riskDetectionService, riskHitService, chatSessionService, chatMessageService) @@ -157,6 +169,7 @@ func main() { openClawConfigService, skillService, externalAccessService, + aiObservabilityService, services.WithInstanceProxyRuntimeRepositories(instanceRepo, runtimePodRepo, bindingRepo), ) systemSettingsHandler := handlers.NewSystemSettingsHandler(systemImageSettingService) @@ -165,15 +178,16 @@ func main() { aiObservabilityHandler := handlers.NewAIObservabilityHandler(aiObservabilityService) riskRuleHandler := handlers.NewRiskRuleHandler(riskRuleService) clusterResourceHandler := handlers.NewClusterResourceHandler(clusterResourceService) - egressProxyHandler := handlers.NewEgressProxyHandler() + egressProxyHandler := handlers.NewEgressProxyHandler(auditEventService) openClawConfigHandler := handlers.NewOpenClawConfigHandler(openClawConfigService) skillHandler := handlers.NewSkillHandler(skillService, instanceService) + skillHubHandler := handlers.NewSkillHubHandler(skillService, instanceService) securityHandler := handlers.NewSecurityHandler(securityScanService) agentHandler := handlers.NewAgentHandler(instanceAgentService, instanceCommandService, instanceRuntimeStatusService, instanceConfigRevisionService, skillService) teamHandler := handlers.NewTeamHandler(teamService) workspaceFileHandler := handlers.NewWorkspaceFileHandler(instanceService, workspaceFileService, runtimeWorkspaceFileService) workspaceFileHandler.SetSkillRepository(skillRepo) - runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, instanceRepo, runtimeEvents) + runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, instanceRepo, runtimeEvents, skillService) // Initialize WebSocket hub and handler wsHub := services.GetHub() @@ -238,6 +252,7 @@ func main() { startBackground := func(ctx context.Context) { log.Printf("Starting leader-only background loops (identity=%s)", cfg.LeaderElection.Identity) syncService.Start() + materializeWorker.Start() teamService.StartBackground(ctx) if runtimeScheduler != nil { runtimeSchedulerMu.Lock() @@ -252,6 +267,7 @@ func main() { } stopBackground := func() { log.Printf("Stopping leader-only background loops (identity=%s)", cfg.LeaderElection.Identity) + materializeWorker.Stop() runtimeSchedulerMu.Lock() if runtimeSchedulerCancel != nil { runtimeSchedulerCancel() @@ -354,6 +370,8 @@ func main() { instances.POST("/:id/restart", instanceHandler.RestartInstance) instances.GET("/:id/status", instanceHandler.GetInstanceStatus) instances.GET("/:id/runtime", instanceHandler.GetRuntimeDetails) + instances.GET("/:id/session-usage", instanceHandler.GetInstanceSessionUsage) + instances.GET("/:id/session-usage/detail", instanceHandler.GetInstanceSessionUsageDetail) instances.POST("/:id/runtime/:command", instanceHandler.CreateRuntimeCommand) instances.GET("/:id/config/revisions", instanceHandler.ListConfigRevisions) instances.POST("/:id/config/revisions/publish", instanceHandler.PublishConfigRevision) @@ -379,11 +397,15 @@ func main() { instances.GET("/:id/skills", skillHandler.ListInstanceSkills) instances.GET("/:id/skills/available", skillHandler.ListAvailableInstanceSkills) instances.POST("/:id/skills", skillHandler.AttachSkillToInstance) + instances.POST("/:id/skills/sync", instanceHandler.RefreshInstanceSkills) + instances.POST("/:id/skills/:skillId/import-to-library", instanceHandler.ImportInstanceSkillToLibrary) + instances.POST("/:id/skills/:skillId/retry-package-collect", instanceHandler.RetrySkillPackageCollect) + instances.POST("/:id/skills/:skillId/publish-to-hub", instanceHandler.PublishInstanceSkillToHub) instances.DELETE("/:id/skills/:skillId", skillHandler.RemoveSkillFromInstance) } // Admin console: cross-user instance listing. Gated by admin - // middleware — non-admin callers get 403. The workspace + // middleware 鈥?non-admin callers get 403. The workspace // /instances endpoint above stays caller-scoped regardless of // role; admin status only unlocks this dedicated surface. adminInstances := api.Group("/admin/instances") @@ -464,6 +486,38 @@ func main() { skills.GET("/:id/scan-results", skillHandler.ListScanResults) } + skillHub := api.Group("/skill-hub") + skillHub.Use(middleware.Auth()) + skillHub.Use(middleware.SetUserInfo(userRepo)) + { + skillHub.GET("/catalog", skillHubHandler.ListCatalog) + skillHub.GET("/tags", skillHubHandler.ListTags) + skillHub.GET("/mine", skillHubHandler.ListMine) + skillHub.GET("/attachable", skillHubHandler.ListAttachable) + skillHub.POST("/skills/import/preview", skillHubHandler.PreviewImportSkills) + skillHub.POST("/skills/import", skillHubHandler.ImportSkills) + skillHub.GET("/skills/:id", skillHubHandler.GetSkill) + skillHub.POST("/skills/:id/publish", skillHubHandler.PublishSkill) + skillHub.POST("/skills/:id/unpublish", skillHubHandler.UnpublishSkill) + skillHub.PUT("/skills/:id/tags", skillHubHandler.UpdateTags) + skillHub.DELETE("/skills/:id", skillHubHandler.DeleteSkill) + skillHub.GET("/skills/:id/download", skillHubHandler.DownloadSkill) + skillHub.POST("/skills/:id/install", skillHubHandler.InstallSkill) + } + + adminSkillHub := api.Group("/admin/skill-hub") + adminSkillHub.Use(middleware.Auth()) + adminSkillHub.Use(middleware.SetUserInfo(userRepo)) + adminSkillHub.Use(middleware.NewAdminAuth(userRepo)) + { + adminSkillHub.GET("/skills", skillHubHandler.ListAdminSkills) + adminSkillHub.POST("/skills/:id/publish", skillHubHandler.PublishSkill) + adminSkillHub.POST("/skills/:id/unpublish", skillHubHandler.UnpublishSkill) + adminSkillHub.PUT("/skills/:id/tags", skillHubHandler.UpdateTags) + adminSkillHub.DELETE("/skills/:id", skillHubHandler.DeleteSkill) + adminSkillHub.POST("/skills/:id/install", skillHubHandler.InstallSkill) + } + systemSettings := api.Group("/system-settings") systemSettings.Use(middleware.Auth()) systemSettings.Use(middleware.SetUserInfo(userRepo)) @@ -509,6 +563,22 @@ func main() { adminCosts.GET("", aiObservabilityHandler.GetCostOverview) } + adminLLMGovernance := api.Group("/admin/llm-governance") + adminLLMGovernance.Use(middleware.Auth()) + adminLLMGovernance.Use(middleware.SetUserInfo(userRepo)) + adminLLMGovernance.Use(middleware.NewAdminAuth(userRepo)) + { + adminLLMGovernance.GET("/overview", aiObservabilityHandler.GetLLMGovernanceOverview) + } + + adminSessionUsage := api.Group("/admin/session-usage") + adminSessionUsage.Use(middleware.Auth()) + adminSessionUsage.Use(middleware.SetUserInfo(userRepo)) + adminSessionUsage.Use(middleware.NewAdminAuth(userRepo)) + { + adminSessionUsage.GET("/overview", aiObservabilityHandler.GetSessionUsageOverview) + } + adminRiskRules := api.Group("/admin/risk-rules") adminRiskRules.Use(middleware.Auth()) adminRiskRules.Use(middleware.SetUserInfo(userRepo)) @@ -559,7 +629,7 @@ func main() { agent.POST("/state/report", agentHandler.ReportState) agent.POST("/skills/inventory", agentHandler.ReportSkillInventory) agent.POST("/skills/upload", agentHandler.UploadSkillPackage) - agent.GET("/skills/versions/:skillVersion/download", skillHandler.DownloadSkillVersionForAgent) + agent.GET("/skills/versions/:skillVersion/download", agentHandler.DownloadSkillVersion) agent.GET("/config/revisions/:id", agentHandler.GetConfigRevision) } diff --git a/backend/internal/aigateway/service.go b/backend/internal/aigateway/service.go index 4fc84b1..61c8621 100644 --- a/backend/internal/aigateway/service.go +++ b/backend/internal/aigateway/service.go @@ -23,6 +23,7 @@ import ( "clawreef/internal/models" "clawreef/internal/repository" "clawreef/internal/services" + "clawreef/internal/utils" ) // ToolCallFunction represents a tool/function call payload. @@ -71,6 +72,8 @@ type ChatCompletionRequest struct { StreamOptions json.RawMessage `json:"stream_options,omitempty"` User *string `json:"user,omitempty"` SessionID *string `json:"session_id,omitempty"` + OpenClawSessionKey *string `json:"-"` + ManagedAgentType *string `json:"-"` InstanceID *int `json:"instance_id,omitempty"` InstanceMode *string `json:"instance_mode,omitempty"` RuntimeType *string `json:"runtime_type,omitempty"` @@ -429,16 +432,43 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr selectedModel: selectedModel, req: req, } - prepared.sessionID = resolveSessionID(req) + sessionSource, sessionID := resolveSessionIdentity(req) + prepared.sessionID = sessionID + if prepared.sessionID != "" && sessionSource != "openclaw_header" { + if runtimeType := strings.TrimSpace(sessionNormalizationRuntimeType(req)); runtimeType != "" { + prepared.sessionID = utils.NormalizeOpenClawSessionID(prepared.sessionID, runtimeType) + } + } prepared.traceID = s.resolveTraceID(userID, req, prepared.sessionID) if prepared.sessionID == "" { prepared.sessionID = normalizeSessionID(nil, prepared.traceID) + sessionSource = "trace_fallback" } prepared.sessionIDPtr = stringPtr(prepared.sessionID) prepared.req.SessionID = prepared.sessionIDPtr prepared.requestID = normalizeOrCreateID(req.RequestID, "req") prepared.requestIDPtr = stringPtr(prepared.requestID) + if sessionSource == "trace_fallback" && req.InstanceID != nil { + if err := s.auditEventService.RecordEvent(&models.AuditEvent{ + TraceID: prepared.traceID, + SessionID: prepared.sessionIDPtr, + RequestID: prepared.requestIDPtr, + UserID: prepared.userIDPtr, + InstanceID: req.InstanceID, + InstanceMode: runtimeAttributionString(req.InstanceMode), + RuntimeType: runtimeAttributionString(req.RuntimeType), + GatewayID: runtimeAttributionString(req.GatewayID), + RuntimePodID: runtimeAttributionInt64(req.RuntimePodID), + EventType: "gateway.session.fallback", + TrafficClass: models.TrafficClassLLM, + Severity: models.AuditSeverityWarn, + Message: fmt.Sprintf("LLM request missing stable session key for instance %d", *req.InstanceID), + }); err != nil { + logPersistenceError("record gateway.session.fallback", prepared.traceID, err) + } + } + sessionTitle := deriveSessionTitle(prepared.req.Messages) if _, err := s.chatSessionService.EnsureSession(prepared.sessionID, prepared.userIDPtr, prepared.req.InstanceID, stringPtr(prepared.traceID), sessionTitle); err != nil { logPersistenceError("ensure chat session", prepared.traceID, err) @@ -2231,13 +2261,23 @@ func fallbackCurrency(currency string) string { } func resolveSessionID(req ChatCompletionRequest) string { + _, sessionID := resolveSessionIdentity(req) + return sessionID +} + +func resolveSessionIdentity(req ChatCompletionRequest) (source string, sessionID string) { + if req.OpenClawSessionKey != nil { + if key := strings.TrimSpace(*req.OpenClawSessionKey); key != "" { + return "openclaw_header", utils.NormalizeOpenClawSessionID(key, sessionNormalizationRuntimeType(req)) + } + } if normalized := normalizeOptionalString(req.SessionID); normalized != "" { - return normalizeExistingIdentifier(normalized, "sess") + return "explicit", normalizeExistingIdentifier(normalized, "sess") } if normalized := normalizeOptionalString(req.User); normalized != "" { - return normalizeExistingIdentifier(normalized, "sess") + return "openai_user", normalizeExistingIdentifier(normalized, "sess") } - return "" + return "", "" } func normalizeSessionID(value *string, traceID string) string { diff --git a/backend/internal/aigateway/session_identity.go b/backend/internal/aigateway/session_identity.go new file mode 100644 index 0000000..1408ecf --- /dev/null +++ b/backend/internal/aigateway/session_identity.go @@ -0,0 +1,63 @@ +package aigateway + +import "strings" + +const defaultManagedSessionKey = "main" + +// IsManagedInstanceType reports whether an instance type participates in managed +// runtime LLM governance (OpenClaw / Hermes). +func IsManagedInstanceType(instanceType string) bool { + switch strings.ToLower(strings.TrimSpace(instanceType)) { + case "openclaw", "hermes": + return true + default: + return false + } +} + +// HasExplicitSessionIdentity returns true when the caller supplied a stable +// session identifier via header or request body fields. +func HasExplicitSessionIdentity(req ChatCompletionRequest) bool { + if req.OpenClawSessionKey != nil && strings.TrimSpace(*req.OpenClawSessionKey) != "" { + return true + } + if normalizeOptionalString(req.SessionID) != "" { + return true + } + if normalizeOptionalString(req.User) != "" { + return true + } + return false +} + +// ApplyManagedInstanceSessionDefaults fills in the default OpenClaw/Hermes session +// key for instance gateway token calls that omitted explicit session identity. +func ApplyManagedInstanceSessionDefaults(req *ChatCompletionRequest, gatewayAuthType, instanceType string) { + if req == nil { + return + } + if strings.TrimSpace(gatewayAuthType) != "instance" { + return + } + instanceType = strings.TrimSpace(instanceType) + if !IsManagedInstanceType(instanceType) { + return + } + req.ManagedAgentType = stringPtr(instanceType) + if HasExplicitSessionIdentity(*req) { + return + } + req.OpenClawSessionKey = stringPtr(defaultManagedSessionKey) +} + +func sessionNormalizationRuntimeType(req ChatCompletionRequest) string { + if req.ManagedAgentType != nil { + if runtimeType := strings.TrimSpace(*req.ManagedAgentType); runtimeType != "" { + return runtimeType + } + } + if req.RuntimeType != nil { + return strings.TrimSpace(*req.RuntimeType) + } + return "" +} diff --git a/backend/internal/aigateway/session_identity_test.go b/backend/internal/aigateway/session_identity_test.go new file mode 100644 index 0000000..f2edc79 --- /dev/null +++ b/backend/internal/aigateway/session_identity_test.go @@ -0,0 +1,61 @@ +package aigateway + +import "testing" + +func TestApplyManagedInstanceSessionDefaultsUsesMainForOpenClawInstanceToken(t *testing.T) { + req := ChatCompletionRequest{ + Messages: []ChatMessage{{Role: "user", Content: "hello"}}, + } + ApplyManagedInstanceSessionDefaults(&req, "instance", "openclaw") + if req.OpenClawSessionKey == nil || *req.OpenClawSessionKey != "main" { + t.Fatalf("expected default session key main, got %+v", req.OpenClawSessionKey) + } + if req.ManagedAgentType == nil || *req.ManagedAgentType != "openclaw" { + t.Fatalf("expected managed agent type openclaw, got %+v", req.ManagedAgentType) + } + if got := resolveSessionID(req); got != "agent:openclaw:main" { + t.Fatalf("expected normalized session id agent:openclaw:main, got %q", got) + } +} + +func TestApplyManagedInstanceSessionDefaultsUsesMainForHermesInstanceToken(t *testing.T) { + req := ChatCompletionRequest{} + ApplyManagedInstanceSessionDefaults(&req, "instance", "hermes") + if got := resolveSessionID(req); got != "agent:hermes:main" { + t.Fatalf("expected normalized session id agent:hermes:main, got %q", got) + } +} + +func TestApplyManagedInstanceSessionDefaultsSkipsUserJWTCalls(t *testing.T) { + req := ChatCompletionRequest{} + ApplyManagedInstanceSessionDefaults(&req, "user", "openclaw") + if req.OpenClawSessionKey != nil { + t.Fatalf("expected no default session key for user auth, got %+v", req.OpenClawSessionKey) + } +} + +func TestApplyManagedInstanceSessionDefaultsRespectsExplicitHeader(t *testing.T) { + explicit := "work" + req := ChatCompletionRequest{ + OpenClawSessionKey: &explicit, + RuntimeType: stringPtr("desktop"), + } + ApplyManagedInstanceSessionDefaults(&req, "instance", "openclaw") + if got := resolveSessionID(req); got != "agent:openclaw:work" { + t.Fatalf("expected explicit session key to win, got %q", got) + } +} + +func TestApplyManagedInstanceSessionDefaultsRespectsExplicitSessionID(t *testing.T) { + explicit := "agent:openclaw:custom" + req := ChatCompletionRequest{ + SessionID: &explicit, + } + ApplyManagedInstanceSessionDefaults(&req, "instance", "openclaw") + if req.OpenClawSessionKey != nil { + t.Fatalf("expected body session id to prevent default header injection, got %+v", req.OpenClawSessionKey) + } + if got := resolveSessionID(req); got != explicit { + t.Fatalf("expected explicit session id %q, got %q", explicit, got) + } +} diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 7d28e8c..95d124d 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -20,6 +20,7 @@ type Config struct { Runtime RuntimePoolConfig `yaml:"runtime"` ObjectStorage ObjectStorageConfig `yaml:"objectStorage"` SkillScanner SkillScannerConfig `yaml:"skillScanner"` + SkillMaterialize SkillMaterializeConfig `yaml:"skillMaterialize"` LeaderElection LeaderElectionConfig `yaml:"leaderElection"` } @@ -200,6 +201,14 @@ type SkillScannerConfig struct { Enabled bool `yaml:"enabled"` } +type SkillMaterializeConfig struct { + Enabled bool `yaml:"enabled"` + TickMS int `yaml:"tickMs"` + BatchSize int `yaml:"batchSize"` + Concurrency int `yaml:"concurrency"` + PerInstanceConcurrency int `yaml:"perInstanceConcurrency"` +} + // Load loads configuration from file and environment variables func Load() (*Config, error) { runtimeNamespace := getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", "clawmanager-system")) @@ -305,6 +314,13 @@ func Load() (*Config, error) { TimeoutSeconds: 30, Enabled: strings.EqualFold(getEnv("SKILL_SCANNER_ENABLED", "false"), "true"), }, + SkillMaterialize: SkillMaterializeConfig{ + Enabled: strings.EqualFold(getEnv("SKILL_MATERIALIZE_WORKER_ENABLED", "true"), "true"), + TickMS: 2000, + BatchSize: 5, + Concurrency: 5, + PerInstanceConcurrency: 2, + }, LeaderElection: LeaderElectionConfig{ Enabled: strings.EqualFold(getEnv("CLAWMANAGER_LEADER_ELECTION", "true"), "true"), Namespace: getEnv("POD_NAMESPACE", "clawmanager-system"), @@ -497,6 +513,21 @@ func applyEnvOverrides(config *Config) { if timeoutSeconds := os.Getenv("SKILL_SCANNER_TIMEOUT_SECONDS"); timeoutSeconds != "" { fmt.Sscanf(timeoutSeconds, "%d", &config.SkillScanner.TimeoutSeconds) } + if enabled := os.Getenv("SKILL_MATERIALIZE_WORKER_ENABLED"); enabled != "" { + config.SkillMaterialize.Enabled = strings.EqualFold(enabled, "true") + } + if tickMS := os.Getenv("SKILL_MATERIALIZE_TICK_MS"); tickMS != "" { + fmt.Sscanf(tickMS, "%d", &config.SkillMaterialize.TickMS) + } + if batchSize := os.Getenv("SKILL_MATERIALIZE_BATCH_SIZE"); batchSize != "" { + fmt.Sscanf(batchSize, "%d", &config.SkillMaterialize.BatchSize) + } + if concurrency := os.Getenv("SKILL_MATERIALIZE_CONCURRENCY"); concurrency != "" { + fmt.Sscanf(concurrency, "%d", &config.SkillMaterialize.Concurrency) + } + if perInstance := os.Getenv("SKILL_MATERIALIZE_PER_INSTANCE_CONCURRENCY"); perInstance != "" { + fmt.Sscanf(perInstance, "%d", &config.SkillMaterialize.PerInstanceConcurrency) + } } func normalizeStorageConfig(config *Config) { diff --git a/backend/internal/db/migrations/023_add_skill_hub.sql b/backend/internal/db/migrations/023_add_skill_hub.sql new file mode 100644 index 0000000..378b822 --- /dev/null +++ b/backend/internal/db/migrations/023_add_skill_hub.sql @@ -0,0 +1,46 @@ +ALTER TABLE skills + ADD COLUMN visibility ENUM('private', 'public') NOT NULL DEFAULT 'private' AFTER status, + ADD COLUMN published_at TIMESTAMP NULL AFTER visibility, + ADD COLUMN published_by INT NULL AFTER published_at, + ADD INDEX idx_skills_visibility (visibility, status, source_type); + +ALTER TABLE skills + ADD CONSTRAINT fk_skills_published_by FOREIGN KEY (published_by) REFERENCES users(id) ON DELETE SET NULL; + +CREATE TABLE IF NOT EXISTS skill_hub_tags ( + id INT AUTO_INCREMENT PRIMARY KEY, + tag_key VARCHAR(64) NOT NULL, + name VARCHAR(120) NOT NULL, + description VARCHAR(255) NULL, + sort_order INT NOT NULL DEFAULT 0, + admin_only BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_skill_hub_tags_tag_key (tag_key), + INDEX idx_skill_hub_tags_sort (sort_order, id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS skill_hub_tag_assignments ( + id INT AUTO_INCREMENT PRIMARY KEY, + skill_id INT NOT NULL, + tag_id INT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES skill_hub_tags(id) ON DELETE CASCADE, + UNIQUE KEY uk_skill_hub_tag_assignments (skill_id, tag_id), + INDEX idx_skill_hub_tag_assignments_tag (tag_id, skill_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +INSERT INTO skill_hub_tags (tag_key, name, description, sort_order, admin_only) VALUES + ('productivity', 'Productivity', 'Efficiency and workflow skills', 10, FALSE), + ('coding', 'Coding', 'Software development skills', 20, FALSE), + ('browser', 'Browser', 'Browser automation skills', 30, FALSE), + ('data', 'Data', 'Data processing and analytics skills', 40, FALSE), + ('communication', 'Communication', 'Messaging and collaboration skills', 50, FALSE), + ('automation', 'Automation', 'Task automation skills', 60, FALSE), + ('research', 'Research', 'Research and information gathering skills', 70, FALSE), + ('community', 'Community', 'Community shared skills', 80, FALSE), + ('admin-curated', 'Admin Curated', 'Curated by platform administrators', 90, TRUE), + ('featured', 'Featured', 'Featured on the Skill Hub', 100, TRUE); + +UPDATE skills SET visibility = 'private'; diff --git a/backend/internal/db/migrations/039_add_skill_package_materialize.sql b/backend/internal/db/migrations/039_add_skill_package_materialize.sql new file mode 100644 index 0000000..352c342 --- /dev/null +++ b/backend/internal/db/migrations/039_add_skill_package_materialize.sql @@ -0,0 +1,71 @@ +SET @instance_skills_workspace_dir_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_skills' + AND COLUMN_NAME = 'workspace_dir' +); +SET @instance_skills_workspace_dir_sql = IF( + @instance_skills_workspace_dir_exists = 0, + 'ALTER TABLE instance_skills ADD COLUMN workspace_dir VARCHAR(120) NULL AFTER install_path', + 'SELECT 1' +); +PREPARE instance_skills_workspace_dir_stmt FROM @instance_skills_workspace_dir_sql; +EXECUTE instance_skills_workspace_dir_stmt; +DEALLOCATE PREPARE instance_skills_workspace_dir_stmt; + +CREATE TABLE IF NOT EXISTS skill_package_materialize_jobs ( + id INT AUTO_INCREMENT PRIMARY KEY, + instance_id INT NOT NULL, + skill_id INT NOT NULL, + blob_id INT NOT NULL, + workspace_dir VARCHAR(120) NOT NULL, + content_hash VARCHAR(128) NOT NULL, + status VARCHAR(30) NOT NULL DEFAULT 'pending', + attempt_count INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL DEFAULT 5, + last_error TEXT NULL, + idempotency_key VARCHAR(255) NOT NULL, + trigger_source VARCHAR(50) NOT NULL DEFAULT 'sync', + started_at TIMESTAMP NULL, + finished_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE, + FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE CASCADE, + UNIQUE KEY uk_sp_materialize_idempotency (idempotency_key), + INDEX idx_sp_materialize_status_created (status, created_at), + INDEX idx_sp_materialize_instance_status (instance_id, status), + INDEX idx_sp_materialize_blob (blob_id, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +UPDATE instance_skills +SET workspace_dir = SUBSTRING_INDEX(REPLACE(install_path, '\\', '/'), '/', -1) +WHERE workspace_dir IS NULL + AND install_path IS NOT NULL + AND TRIM(install_path) <> ''; + +UPDATE instance_commands ic +JOIN instances i ON i.id = ic.instance_id +SET ic.status = 'cancelled', + ic.error_message = 'superseded by skill_package_materialize_jobs' +WHERE ic.command_type = 'collect_skill_package' + AND ic.status IN ('pending', 'dispatched', 'running') + AND (LOWER(TRIM(i.instance_mode)) = 'lite' OR LOWER(TRIM(i.runtime_type)) = 'gateway'); + +SET @instance_skills_workspace_dir_index_exists = ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_skills' + AND INDEX_NAME = 'idx_instance_skills_workspace_dir' +); +SET @instance_skills_workspace_dir_index_sql = IF( + @instance_skills_workspace_dir_index_exists = 0, + 'ALTER TABLE instance_skills ADD INDEX idx_instance_skills_workspace_dir (workspace_dir)', + 'SELECT 1' +); +PREPARE instance_skills_workspace_dir_index_stmt FROM @instance_skills_workspace_dir_index_sql; +EXECUTE instance_skills_workspace_dir_index_stmt; +DEALLOCATE PREPARE instance_skills_workspace_dir_index_stmt; diff --git a/backend/internal/db/migrations/040_fix_injected_skill_provenance.sql b/backend/internal/db/migrations/040_fix_injected_skill_provenance.sql new file mode 100644 index 0000000..1230641 --- /dev/null +++ b/backend/internal/db/migrations/040_fix_injected_skill_provenance.sql @@ -0,0 +1,7 @@ +UPDATE instance_skills isk +JOIN skills s ON s.id = isk.skill_id +SET isk.source_type = 'injected_by_clawmanager' +WHERE isk.source_type = 'discovered_in_instance' + AND s.source_type = 'uploaded' + AND s.visibility = 'public' + AND isk.status = 'active'; diff --git a/backend/internal/db/migrations/041_add_session_usage_indexes.sql b/backend/internal/db/migrations/041_add_session_usage_indexes.sql new file mode 100644 index 0000000..6044542 --- /dev/null +++ b/backend/internal/db/migrations/041_add_session_usage_indexes.sql @@ -0,0 +1,52 @@ +SET @dbname = DATABASE(); + +SET @indexname = 'idx_cost_records_instance_id'; +SET @preparedStatement = ( + SELECT IF( + EXISTS( + SELECT 1 FROM information_schema.statistics + WHERE table_schema = @dbname + AND table_name = 'cost_records' + AND index_name = @indexname + ), + 'SELECT 1', + 'ALTER TABLE cost_records ADD INDEX idx_cost_records_instance_id (instance_id)' + ) +); +PREPARE stmt FROM @preparedStatement; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @indexname = 'idx_cost_records_session_id'; +SET @preparedStatement = ( + SELECT IF( + EXISTS( + SELECT 1 FROM information_schema.statistics + WHERE table_schema = @dbname + AND table_name = 'cost_records' + AND index_name = @indexname + ), + 'SELECT 1', + 'ALTER TABLE cost_records ADD INDEX idx_cost_records_session_id (session_id)' + ) +); +PREPARE stmt FROM @preparedStatement; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @indexname = 'idx_model_invocations_instance_session'; +SET @preparedStatement = ( + SELECT IF( + EXISTS( + SELECT 1 FROM information_schema.statistics + WHERE table_schema = @dbname + AND table_name = 'model_invocations' + AND index_name = @indexname + ), + 'SELECT 1', + 'ALTER TABLE model_invocations ADD INDEX idx_model_invocations_instance_session (instance_id, session_id, created_at)' + ) +); +PREPARE stmt FROM @preparedStatement; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend/internal/db/migrations_test.go b/backend/internal/db/migrations_test.go index ab5768c..c6dcd12 100644 --- a/backend/internal/db/migrations_test.go +++ b/backend/internal/db/migrations_test.go @@ -183,3 +183,20 @@ func TestMigration038AddsGatewayTokenAliases(t *testing.T) { t.Fatalf("migration 038 must not store raw access tokens") } } + +func TestMigration041AddsSessionUsageIndexes(t *testing.T) { + raw, err := embeddedMigrations.ReadFile("migrations/041_add_session_usage_indexes.sql") + if err != nil { + t.Fatalf("read migration 041: %v", err) + } + sql := string(raw) + for _, required := range []string{ + "idx_cost_records_instance_id", + "idx_cost_records_session_id", + "idx_model_invocations_instance_session", + } { + if !strings.Contains(sql, required) { + t.Fatalf("migration 041 must contain %s", required) + } + } +} diff --git a/backend/internal/egresspolicy/policy.go b/backend/internal/egresspolicy/policy.go new file mode 100644 index 0000000..c0e66fd --- /dev/null +++ b/backend/internal/egresspolicy/policy.go @@ -0,0 +1,118 @@ +package egresspolicy + +import ( + "os" + "strings" +) + +type Mode string + +const ( + ModeOpen Mode = "open" + ModeDenylist Mode = "denylist" + ModeAllowlist Mode = "allowlist" +) + +type Policy struct { + Mode Mode + DeniedHostSuffixes []string + AllowedHostSuffixes []string +} + +func LoadFromEnv() Policy { + mode := Mode(strings.ToLower(strings.TrimSpace(os.Getenv("CLAWMANAGER_EGRESS_LLM_POLICY")))) + if mode == "" { + mode = ModeDenylist + } + + policy := Policy{ + Mode: mode, + DeniedHostSuffixes: append(defaultDeniedHostSuffixes(), + splitCSV(os.Getenv("CLAWMANAGER_EGRESS_DENIED_SUFFIXES"))...), + AllowedHostSuffixes: append(defaultAllowedHostSuffixes(), + splitCSV(os.Getenv("CLAWMANAGER_EGRESS_ALLOWED_SUFFIXES"))...), + } + return policy +} + +func defaultDeniedHostSuffixes() []string { + return []string{ + "api.openai.com", + "openai.azure.com", + "api.anthropic.com", + "generativelanguage.googleapis.com", + "api.deepseek.com", + "api.moonshot.cn", + "open.bigmodel.cn", + "dashscope.aliyuncs.com", + } +} + +func defaultAllowedHostSuffixes() []string { + return []string{ + "github.com", + "registry-1.docker.io", + "pypi.org", + "npmjs.org", + "clawmanager-gateway", + "clawmanager-egress-proxy", + } +} + +func (p Policy) AllowHost(host string) (bool, string) { + host = normalizeHost(host) + if host == "" { + return false, "empty host" + } + + switch p.Mode { + case ModeOpen, "": + return true, "" + case ModeAllowlist: + if matchesAnySuffix(host, p.AllowedHostSuffixes) { + return true, "" + } + return false, "host not in allowlist" + default: + if matchesAnySuffix(host, p.DeniedHostSuffixes) { + return false, "llm provider host blocked" + } + return true, "" + } +} + +func normalizeHost(host string) string { + host = strings.TrimSpace(strings.ToLower(host)) + if host == "" { + return "" + } + if idx := strings.Index(host, ":"); idx >= 0 { + host = host[:idx] + } + return strings.Trim(host, ".") +} + +func matchesAnySuffix(host string, suffixes []string) bool { + for _, suffix := range suffixes { + suffix = strings.TrimSpace(strings.ToLower(suffix)) + if suffix == "" { + continue + } + if host == suffix || strings.HasSuffix(host, "."+suffix) { + return true + } + } + return false +} + +func splitCSV(raw string) []string { + parts := strings.Split(raw, ",") + result := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + result = append(result, part) + } + } + return result +} diff --git a/backend/internal/egresspolicy/policy_test.go b/backend/internal/egresspolicy/policy_test.go new file mode 100644 index 0000000..f1ccc3b --- /dev/null +++ b/backend/internal/egresspolicy/policy_test.go @@ -0,0 +1,33 @@ +package egresspolicy + +import "testing" + +func TestPolicyDenylistBlocksOpenAI(t *testing.T) { + policy := Policy{ + Mode: ModeDenylist, + DeniedHostSuffixes: defaultDeniedHostSuffixes(), + } + allowed, reason := policy.AllowHost("api.openai.com") + if allowed || reason == "" { + t.Fatalf("expected openai host to be blocked, allowed=%v reason=%q", allowed, reason) + } +} + +func TestPolicyDenylistAllowsGitHub(t *testing.T) { + policy := Policy{ + Mode: ModeDenylist, + DeniedHostSuffixes: defaultDeniedHostSuffixes(), + } + allowed, reason := policy.AllowHost("github.com") + if !allowed || reason != "" { + t.Fatalf("expected github host to be allowed, allowed=%v reason=%q", allowed, reason) + } +} + +func TestPolicyOpenAllowsEverything(t *testing.T) { + policy := Policy{Mode: ModeOpen} + allowed, reason := policy.AllowHost("api.openai.com") + if !allowed || reason != "" { + t.Fatalf("expected open mode to allow host, allowed=%v reason=%q", allowed, reason) + } +} diff --git a/backend/internal/handlers/agent_handler.go b/backend/internal/handlers/agent_handler.go index b2d1c6f..849ed48 100644 --- a/backend/internal/handlers/agent_handler.go +++ b/backend/internal/handlers/agent_handler.go @@ -1,6 +1,7 @@ package handlers import ( + "fmt" "net/http" "strconv" "strings" @@ -198,6 +199,7 @@ func (h *AgentHandler) ReportSkillInventory(c *gin.Context) { utils.HandleError(c, err) return } + h.skillService.CompletePendingSkillInventorySync(session.Instance.ID) utils.Success(c, http.StatusOK, "Agent skill inventory reported successfully", nil) } @@ -225,12 +227,26 @@ func (h *AgentHandler) UploadSkillPackage(c *gin.Context) { } item, err := h.skillService.UploadAgentSkillPackage(c.Request.Context(), session.Instance.ID, req, fileHeader) if err != nil { - utils.HandleError(c, err) + utils.HandleHubError(c, err) return } utils.Success(c, http.StatusCreated, "Agent skill package uploaded successfully", item) } +func (h *AgentHandler) DownloadSkillVersion(c *gin.Context) { + if _, ok := h.authenticateAgentSession(c); !ok { + return + } + content, fileName, err := h.skillService.DownloadSkillVersionByExternalID(c.Param("skillVersion")) + if err != nil { + utils.HandleError(c, err) + return + } + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName)) + c.Data(http.StatusOK, "application/octet-stream", content) +} + func (h *AgentHandler) authenticateAgentSession(c *gin.Context) (*services.AgentSession, bool) { sessionToken := extractBearerToken(c.GetHeader("Authorization")) if sessionToken == "" { diff --git a/backend/internal/handlers/agent_handler_test.go b/backend/internal/handlers/agent_handler_test.go new file mode 100644 index 0000000..c35b4c3 --- /dev/null +++ b/backend/internal/handlers/agent_handler_test.go @@ -0,0 +1,25 @@ +package handlers + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestDownloadSkillVersionRequiresAgentSession(t *testing.T) { + gin.SetMode(gin.TestMode) + handler := NewAgentHandler(nil, nil, nil, nil, nil) + + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/agent/skills/versions/skill-version-1/download", nil) + c.Params = gin.Params{{Key: "skillVersion", Value: "skill-version-1"}} + + handler.DownloadSkillVersion(c) + + if recorder.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusUnauthorized) + } +} diff --git a/backend/internal/handlers/ai_gateway_handler.go b/backend/internal/handlers/ai_gateway_handler.go index 28e77d0..16cf181 100644 --- a/backend/internal/handlers/ai_gateway_handler.go +++ b/backend/internal/handlers/ai_gateway_handler.go @@ -49,8 +49,9 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) { return } req.RawBody = rawBody - if req.SessionID == nil { - if sessionKey := strings.TrimSpace(c.GetHeader("x-openclaw-session-key")); sessionKey != "" { + if sessionKey := strings.TrimSpace(c.GetHeader("x-openclaw-session-key")); sessionKey != "" { + req.OpenClawSessionKey = &sessionKey + if req.SessionID == nil { req.SessionID = &sessionKey } } @@ -59,6 +60,13 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) { req.TraceID = &runID } } + gatewayAuthType, _ := c.Get("gatewayAuthType") + instanceType, _ := c.Get("instanceType") + aigateway.ApplyManagedInstanceSessionDefaults( + &req, + stringValue(gatewayAuthType), + stringValue(instanceType), + ) userID, exists := c.Get("userID") if !exists { @@ -158,3 +166,11 @@ func setInt64Metadata(c *gin.Context, field **int64, key string) bool { } return **field == value } + +func stringValue(raw interface{}) string { + value, ok := raw.(string) + if !ok { + return "" + } + return strings.TrimSpace(value) +} diff --git a/backend/internal/handlers/ai_observability_handler.go b/backend/internal/handlers/ai_observability_handler.go index 733cec7..52f57d8 100644 --- a/backend/internal/handlers/ai_observability_handler.go +++ b/backend/internal/handlers/ai_observability_handler.go @@ -30,6 +30,15 @@ type CostQueryRequest struct { Search string `form:"search"` } +// SessionUsageOverviewQueryRequest binds admin session usage overview filters. +type SessionUsageOverviewQueryRequest struct { + Page int `form:"page,default=1"` + Limit int `form:"limit,default=20"` + Search string `form:"search"` + Since string `form:"since"` + Until string `form:"until"` +} + // NewAIObservabilityHandler creates a new observability handler. func NewAIObservabilityHandler(service services.AIObservabilityService) *AIObservabilityHandler { return &AIObservabilityHandler{service: service} @@ -90,3 +99,51 @@ func (h *AIObservabilityHandler) GetCostOverview(c *gin.Context) { utils.Success(c, http.StatusOK, "AI cost overview retrieved successfully", overview) } + +// GetLLMGovernanceOverview returns managed-runtime LLM governance summary for admins. +func (h *AIObservabilityHandler) GetLLMGovernanceOverview(c *gin.Context) { + overview, err := h.service.GetLLMGovernanceOverview() + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "LLM governance overview retrieved successfully", overview) +} + +// GetSessionUsageOverview returns cross-instance session token usage for admins. +func (h *AIObservabilityHandler) GetSessionUsageOverview(c *gin.Context) { + var req SessionUsageOverviewQueryRequest + if err := c.ShouldBindQuery(&req); err != nil { + utils.ValidationError(c, err) + return + } + since, err := parseOptionalRFC3339(req.Since) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid since timestamp") + return + } + until, parseUntilErr := parseOptionalRFC3339(req.Until) + if parseUntilErr != nil { + utils.Error(c, http.StatusBadRequest, "Invalid until timestamp") + return + } + if err := validateSessionUsageTimeRange(since, until); err != nil { + utils.Error(c, http.StatusBadRequest, err.Error()) + return + } + + overview, err := h.service.GetAdminSessionUsageOverview(services.InstanceSessionUsageOverviewQuery{ + Page: req.Page, + Limit: req.Limit, + Search: req.Search, + Since: since, + Until: until, + }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Session usage overview retrieved successfully", overview) +} diff --git a/backend/internal/handlers/ai_observability_handler_test.go b/backend/internal/handlers/ai_observability_handler_test.go new file mode 100644 index 0000000..a474bdc --- /dev/null +++ b/backend/internal/handlers/ai_observability_handler_test.go @@ -0,0 +1,105 @@ +package handlers + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "clawreef/internal/repository" + "clawreef/internal/services" + + "github.com/gin-gonic/gin" +) + +type stubAIObservabilityHandlerService struct { + overview *services.InstanceSessionUsageOverview + err error + lastQuery services.InstanceSessionUsageOverviewQuery +} + +func (s *stubAIObservabilityHandlerService) ListAuditItems(services.AuditQuery) (*services.AuditListResult, error) { + return nil, nil +} +func (s *stubAIObservabilityHandlerService) GetTraceDetail(string) (*services.AuditTraceDetail, error) { + return nil, nil +} +func (s *stubAIObservabilityHandlerService) GetCostOverview(services.CostQuery) (*services.CostOverview, error) { + return nil, nil +} +func (s *stubAIObservabilityHandlerService) GetInstanceSessionUsage(int, services.InstanceSessionUsageQuery) (*services.InstanceSessionUsageResult, error) { + return nil, nil +} +func (s *stubAIObservabilityHandlerService) GetInstanceSessionUsageDetail(int, string, repository.SessionUsageFilter) (*services.InstanceSessionUsageDetail, error) { + return nil, nil +} +func (s *stubAIObservabilityHandlerService) GetInstanceLLMGovernanceStatus(int, map[string]interface{}) (*services.InstanceLLMGovernanceStatus, error) { + return nil, nil +} +func (s *stubAIObservabilityHandlerService) GetLLMGovernanceOverview() (*services.LLMGovernanceOverview, error) { + return nil, nil +} +func (s *stubAIObservabilityHandlerService) GetAdminSessionUsageOverview(query services.InstanceSessionUsageOverviewQuery) (*services.InstanceSessionUsageOverview, error) { + s.lastQuery = query + if s.err != nil { + return nil, s.err + } + return s.overview, nil +} + +func TestGetSessionUsageOverviewReturns200(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/admin/session-usage/overview?page=1&limit=10&search=oc", nil) + + service := &stubAIObservabilityHandlerService{ + overview: &services.InstanceSessionUsageOverview{ + Summary: services.InstanceSessionUsageSummary{Currency: "USD"}, + Items: []services.InstanceSessionUsageOverviewItem{}, + Total: 0, + Page: 1, + Limit: 10, + }, + } + handler := NewAIObservabilityHandler(service) + handler.GetSessionUsageOverview(c) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + if service.lastQuery.Search != "oc" { + t.Fatalf("expected search=oc, got %q", service.lastQuery.Search) + } + if !strings.Contains(recorder.Body.String(), "Session usage overview retrieved successfully") { + t.Fatalf("unexpected body: %s", recorder.Body.String()) + } +} + +func TestGetSessionUsageOverviewInvalidSince(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/admin/session-usage/overview?since=not-a-date", nil) + + handler := NewAIObservabilityHandler(&stubAIObservabilityHandlerService{}) + handler.GetSessionUsageOverview(c) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String()) + } +} + +func TestGetSessionUsageOverviewRejectsUntilBeforeSince(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/admin/session-usage/overview?since=2026-07-10T00:00:00Z&until=2026-07-01T00:00:00Z", nil) + + handler := NewAIObservabilityHandler(&stubAIObservabilityHandlerService{}) + handler.GetSessionUsageOverview(c) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String()) + } +} diff --git a/backend/internal/handlers/egress_proxy_handler.go b/backend/internal/handlers/egress_proxy_handler.go index 602e474..9c2cfce 100644 --- a/backend/internal/handlers/egress_proxy_handler.go +++ b/backend/internal/handlers/egress_proxy_handler.go @@ -1,22 +1,31 @@ package handlers import ( + "fmt" "io" + "log" "net" "net/http" + "strconv" "strings" "time" + "clawreef/internal/egresspolicy" + "clawreef/internal/models" + "clawreef/internal/services" + "github.com/gin-gonic/gin" ) // EgressProxyHandler provides a minimal forward proxy for ordinary HTTP/HTTPS traffic. type EgressProxyHandler struct { transport *http.Transport + policy egresspolicy.Policy + audit services.AuditEventService } // NewEgressProxyHandler creates a new egress proxy handler. -func NewEgressProxyHandler() *EgressProxyHandler { +func NewEgressProxyHandler(audit services.AuditEventService) *EgressProxyHandler { return &EgressProxyHandler{ transport: &http.Transport{ Proxy: nil, @@ -27,6 +36,8 @@ func NewEgressProxyHandler() *EgressProxyHandler { TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, }, + policy: egresspolicy.LoadFromEnv(), + audit: audit, } } @@ -42,6 +53,12 @@ func (h *EgressProxyHandler) Handle(c *gin.Context) { return } + if allowed, reason := h.policy.AllowHost(c.Request.URL.Host); !allowed { + h.recordBlockedEgress(c, c.Request.URL.Host, reason) + c.String(http.StatusForbidden, "egress blocked: %s (%s)", c.Request.URL.Host, reason) + return + } + outReq := c.Request.Clone(c.Request.Context()) outReq.RequestURI = "" removeHopHeaders(outReq.Header) @@ -66,6 +83,12 @@ func (h *EgressProxyHandler) handleConnect(c *gin.Context) { return } + if allowed, reason := h.policy.AllowHost(target); !allowed { + h.recordBlockedEgress(c, target, reason) + c.String(http.StatusForbidden, "egress blocked: %s (%s)", target, reason) + return + } + upstreamConn, err := net.DialTimeout("tcp", target, 30*time.Second) if err != nil { c.String(http.StatusBadGateway, "proxy connect error: %v", err) @@ -91,6 +114,41 @@ func (h *EgressProxyHandler) handleConnect(c *gin.Context) { go tunnelConns(clientConn, upstreamConn) } +func (h *EgressProxyHandler) recordBlockedEgress(c *gin.Context, host, reason string) { + if h.audit == nil { + return + } + instanceID := resolveEgressInstanceID(c) + remoteAddr := strings.TrimSpace(c.Request.RemoteAddr) + message := fmt.Sprintf("Blocked egress to %s (%s) from %s", host, reason, remoteAddr) + if err := h.audit.RecordEvent(&models.AuditEvent{ + TraceID: fmt.Sprintf("egress_%d", time.Now().UnixNano()), + InstanceID: instanceID, + EventType: "egress.llm.blocked", + TrafficClass: models.TrafficClassGenericEgress, + Severity: models.AuditSeverityWarn, + Message: message, + }); err != nil { + log.Printf("failed to record egress block audit event: %v", err) + } +} + +func resolveEgressInstanceID(c *gin.Context) *int { + for _, headerName := range []string{ + "X-ClawManager-Instance-Id", + "X-ClawManager-Egress-Instance-Id", + } { + raw := strings.TrimSpace(c.GetHeader(headerName)) + if raw == "" { + continue + } + if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 { + return &parsed + } + } + return nil +} + func tunnelConns(dst net.Conn, src net.Conn) { defer dst.Close() defer src.Close() diff --git a/backend/internal/handlers/egress_proxy_handler_test.go b/backend/internal/handlers/egress_proxy_handler_test.go new file mode 100644 index 0000000..367d217 --- /dev/null +++ b/backend/internal/handlers/egress_proxy_handler_test.go @@ -0,0 +1,82 @@ +package handlers + +import ( + "net/http" + "net/http/httptest" + "testing" + + "clawreef/internal/egresspolicy" + "clawreef/internal/models" + + "github.com/gin-gonic/gin" +) + +type stubEgressAuditService struct { + events []*models.AuditEvent +} + +func (s *stubEgressAuditService) RecordEvent(event *models.AuditEvent) error { + s.events = append(s.events, event) + return nil +} + +func (s *stubEgressAuditService) ListEventsByTraceID(string) ([]models.AuditEvent, error) { + return nil, nil +} + +func TestEgressProxyHandlerBlocksDeniedConnectHost(t *testing.T) { + gin.SetMode(gin.TestMode) + audit := &stubEgressAuditService{} + handler := &EgressProxyHandler{ + policy: egresspolicy.Policy{ + Mode: egresspolicy.ModeDenylist, + DeniedHostSuffixes: []string{"api.openai.com"}, + }, + audit: audit, + } + + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodConnect, "https://api.openai.com:443", nil) + ctx.Request.Host = "api.openai.com:443" + ctx.Request.Header.Set("X-ClawManager-Instance-Id", "42") + + handler.handleConnect(ctx) + + if recorder.Code != http.StatusForbidden { + t.Fatalf("expected 403, got %d", recorder.Code) + } + if len(audit.events) != 1 || audit.events[0].EventType != "egress.llm.blocked" { + t.Fatalf("expected egress audit event, got %+v", audit.events) + } + if audit.events[0].InstanceID == nil || *audit.events[0].InstanceID != 42 { + t.Fatalf("expected instance id 42 on egress audit event, got %+v", audit.events[0].InstanceID) + } +} + +func TestEgressProxyHandlerAcceptsEgressInstanceHeaderAlias(t *testing.T) { + gin.SetMode(gin.TestMode) + audit := &stubEgressAuditService{} + handler := &EgressProxyHandler{ + policy: egresspolicy.Policy{ + Mode: egresspolicy.ModeDenylist, + DeniedHostSuffixes: []string{"api.openai.com"}, + }, + audit: audit, + } + + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodConnect, "https://api.openai.com:443", nil) + ctx.Request.Host = "api.openai.com:443" + ctx.Request.Header.Set("X-ClawManager-Egress-Instance-Id", "77") + + handler.handleConnect(ctx) + + if recorder.Code != http.StatusForbidden { + t.Fatalf("expected 403, got %d", recorder.Code) + } + if audit.events[0].InstanceID == nil || *audit.events[0].InstanceID != 77 { + t.Fatalf("expected instance id 77 on egress audit event, got %+v", audit.events[0].InstanceID) + } +} diff --git a/backend/internal/handlers/instance_handler.go b/backend/internal/handlers/instance_handler.go index c1ef978..e2b297a 100644 --- a/backend/internal/handlers/instance_handler.go +++ b/backend/internal/handlers/instance_handler.go @@ -13,6 +13,7 @@ import ( "time" "clawreef/internal/models" + "clawreef/internal/repository" "clawreef/internal/services" "clawreef/internal/utils" @@ -135,10 +136,11 @@ type InstanceHandler struct { openClawConfigService services.OpenClawConfigService skillService services.SkillService externalAccessService services.InstanceExternalAccessService + aiObservabilityService services.AIObservabilityService } // NewInstanceHandler creates a new instance handler -func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService, externalAccessService services.InstanceExternalAccessService, proxyOptions ...services.InstanceProxyServiceOption) *InstanceHandler { +func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService, externalAccessService services.InstanceExternalAccessService, aiObservabilityService services.AIObservabilityService, proxyOptions ...services.InstanceProxyServiceOption) *InstanceHandler { accessService := services.NewInstanceAccessService() return &InstanceHandler{ instanceService: instanceService, @@ -153,6 +155,7 @@ func NewInstanceHandler(instanceService services.InstanceService, instanceAgentS openClawConfigService: openClawConfigService, skillService: skillService, externalAccessService: externalAccessService, + aiObservabilityService: aiObservabilityService, } } @@ -164,9 +167,10 @@ func (h *InstanceHandler) Shutdown() { } type InstanceRuntimeDetailsResponse struct { - Runtime *services.InstanceRuntimeStatusPayload `json:"runtime,omitempty"` - Agent *services.InstanceAgentPayload `json:"agent,omitempty"` - Commands []services.InstanceCommandPayload `json:"commands,omitempty"` + Runtime *services.InstanceRuntimeStatusPayload `json:"runtime,omitempty"` + Agent *services.InstanceAgentPayload `json:"agent,omitempty"` + Commands []services.InstanceCommandPayload `json:"commands,omitempty"` + LLMGovernance *services.InstanceLLMGovernanceStatus `json:"llm_governance,omitempty"` } type CreateRuntimeCommandRequest struct { @@ -364,9 +368,10 @@ func (h *InstanceHandler) CreateInstance(c *gin.Context) { return } + userRole, _ := c.Get("userRole") for _, skillID := range skillIDs { - if _, err := h.skillService.AttachSkillToInstance(instance.ID, skillID); err != nil { - utils.HandleError(c, err) + if _, err := h.skillService.AttachSkillToInstance(userID.(int), userRole.(string), instance.ID, skillID); err != nil { + utils.HandleHubError(c, err) return } } @@ -400,6 +405,7 @@ func instanceCreateRequestToService(req CreateInstanceRequest) services.CreateIn func (h *InstanceHandler) BatchCreateLiteInstances(c *gin.Context) { userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") var req BatchCreateLiteInstancesRequest if err := c.ShouldBindJSON(&req); err != nil { @@ -449,7 +455,7 @@ func (h *InstanceHandler) BatchCreateLiteInstances(c *gin.Context) { } attachFailed := false for _, skillID := range skillIDs { - if _, err := h.skillService.AttachSkillToInstance(instance.ID, skillID); err != nil { + if _, err := h.skillService.AttachSkillToInstance(userID.(int), userRole.(string), instance.ID, skillID); err != nil { result.Status = "failed" result.Error = err.Error() response.Failed++ @@ -986,7 +992,7 @@ func availabilityForInstanceStatus(status string) string { } func (h *InstanceHandler) GetRuntimeDetails(c *gin.Context) { - id, _, ok := h.resolveOwnedInstance(c) + id, instance, ok := h.resolveOwnedInstance(c) if !ok { return } @@ -1007,11 +1013,128 @@ func (h *InstanceHandler) GetRuntimeDetails(c *gin.Context) { return } - utils.Success(c, http.StatusOK, "Instance runtime details retrieved successfully", InstanceRuntimeDetailsResponse{ + response := InstanceRuntimeDetailsResponse{ Runtime: runtime, Agent: agent, Commands: commands, + } + if h.aiObservabilityService != nil && instance != nil && + (instance.Type == "openclaw" || instance.Type == "hermes") { + var systemInfo map[string]interface{} + if runtime != nil { + systemInfo = runtime.SystemInfo + } + if governance, govErr := h.aiObservabilityService.GetInstanceLLMGovernanceStatus(id, systemInfo); govErr == nil { + response.LLMGovernance = governance + } + } + + utils.Success(c, http.StatusOK, "Instance runtime details retrieved successfully", response) +} + +type SessionUsageQueryRequest struct { + Page int `form:"page,default=1"` + Limit int `form:"limit,default=20"` + Search string `form:"search"` + Since string `form:"since"` + Until string `form:"until"` +} + +type SessionUsageDetailQueryRequest struct { + SessionID string `form:"session_id" binding:"required"` + Since string `form:"since"` + Until string `form:"until"` +} + +func (h *InstanceHandler) GetInstanceSessionUsage(c *gin.Context) { + id, _, ok := h.resolveOwnedInstance(c) + if !ok { + return + } + if h.aiObservabilityService == nil { + utils.Error(c, http.StatusInternalServerError, "Session usage service is not configured") + return + } + + var req SessionUsageQueryRequest + if err := c.ShouldBindQuery(&req); err != nil { + utils.ValidationError(c, err) + return + } + since, err := parseOptionalRFC3339(req.Since) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid since timestamp") + return + } + until, parseUntilErr := parseOptionalRFC3339(req.Until) + if parseUntilErr != nil { + utils.Error(c, http.StatusBadRequest, "Invalid until timestamp") + return + } + if err := validateSessionUsageTimeRange(since, until); err != nil { + utils.Error(c, http.StatusBadRequest, err.Error()) + return + } + + result, err := h.aiObservabilityService.GetInstanceSessionUsage(id, services.InstanceSessionUsageQuery{ + Page: req.Page, + Limit: req.Limit, + Search: req.Search, + Since: since, + Until: until, }) + if err != nil { + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Instance session usage retrieved successfully", result) +} + +func (h *InstanceHandler) GetInstanceSessionUsageDetail(c *gin.Context) { + id, _, ok := h.resolveOwnedInstance(c) + if !ok { + return + } + if h.aiObservabilityService == nil { + utils.Error(c, http.StatusInternalServerError, "Session usage service is not configured") + return + } + + var req SessionUsageDetailQueryRequest + if err := c.ShouldBindQuery(&req); err != nil { + utils.ValidationError(c, err) + return + } + since, err := parseOptionalRFC3339(req.Since) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid since timestamp") + return + } + until, parseUntilErr := parseOptionalRFC3339(req.Until) + if parseUntilErr != nil { + utils.Error(c, http.StatusBadRequest, "Invalid until timestamp") + return + } + if err := validateSessionUsageTimeRange(since, until); err != nil { + utils.Error(c, http.StatusBadRequest, err.Error()) + return + } + + detail, err := h.aiObservabilityService.GetInstanceSessionUsageDetail(id, req.SessionID, repository.SessionUsageFilter{ + Since: since, + Until: until, + }) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "not found") { + utils.Error(c, http.StatusNotFound, "Session usage not found") + return + } + utils.HandleError(c, err) + return + } + + utils.Success(c, http.StatusOK, "Instance session usage detail retrieved successfully", detail) } func (h *InstanceHandler) CreateRuntimeCommand(c *gin.Context) { @@ -1143,25 +1266,37 @@ func (h *InstanceHandler) resolveOwnedInstance(c *gin.Context) (int, *models.Ins utils.Error(c, http.StatusBadRequest, "Invalid instance ID") return 0, nil, false } + instance, ok := h.authorizeInstanceAccess(c, id) + if !ok { + return 0, nil, false + } + return id, instance, true +} - instance, err := h.instanceService.GetByID(id) +func (h *InstanceHandler) authorizeInstanceAccess(c *gin.Context, instanceID int) (*models.Instance, bool) { + if instanceID <= 0 { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return nil, false + } + + instance, err := h.instanceService.GetByID(instanceID) if err != nil { utils.HandleError(c, err) - return 0, nil, false + return nil, false } if instance == nil { utils.Error(c, http.StatusNotFound, "Instance not found") - return 0, nil, false + return nil, false } userID, _ := c.Get("userID") userRole, _ := c.Get("userRole") if userRole != "admin" && instance.UserID != userID.(int) { utils.Error(c, http.StatusForbidden, "Access denied") - return 0, nil, false + return nil, false } - return id, instance, true + return instance, true } // GenerateAccessToken generates an access token for an instance @@ -1634,6 +1769,110 @@ func (h *InstanceHandler) ImportHermes(c *gin.Context) { utils.Success(c, http.StatusOK, "Hermes workspace imported successfully", nil) } +func (h *InstanceHandler) RefreshInstanceSkills(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + if instance.Type != "openclaw" && instance.Type != "hermes" { + utils.Error(c, http.StatusBadRequest, "skill inventory sync is only available for openclaw and hermes instances") + return + } + userID, _ := c.Get("userID") + issuedBy := userID.(int) + command, err := h.createSkillInventorySyncCommand(instance, issuedBy) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill inventory sync requested", command) +} + +func (h *InstanceHandler) createSkillInventorySyncCommand(instance *models.Instance, issuedBy int) (*services.InstanceCommandPayload, error) { + command, err := h.instanceCommandService.Create(instance.ID, &issuedBy, services.CreateInstanceCommandRequest{ + CommandType: services.InstanceCommandTypeSyncSkillInventory, + Payload: map[string]interface{}{ + "trigger": "manual", + "mode": "full", + }, + IdempotencyKey: fmt.Sprintf("sync-skill-inventory-%d-%d", instance.ID, time.Now().Unix()), + TimeoutSeconds: 300, + }) + if err != nil { + return nil, err + } + if services.IsLiteRuntimeInstance(instance) || services.SupportsServerWorkspaceSkillScan(instance) { + if err := h.skillService.RequestLiteSkillInventorySync(instance.ID); err != nil { + return nil, err + } + } + return command, nil +} + +func (h *InstanceHandler) ImportInstanceSkillToLibrary(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + skillID, err := strconv.Atoi(c.Param("skillId")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + item, err := h.skillService.ImportInstanceSkillToLibrary(userID.(int), userRole.(string), instance.ID, skillID) + if err != nil { + utils.HandleHubError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill imported to library successfully", item) +} + +func (h *InstanceHandler) RetrySkillPackageCollect(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + skillID, err := strconv.Atoi(c.Param("skillId")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + if err := h.skillService.RetrySkillPackageCollection(userID.(int), userRole.(string), instance.ID, skillID); err != nil { + utils.HandleHubError(c, err) + return + } + utils.Success(c, http.StatusAccepted, "Skill package collection requested", gin.H{"status": "pending"}) +} + +func (h *InstanceHandler) PublishInstanceSkillToHub(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + skillID, err := strconv.Atoi(c.Param("skillId")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + var req services.PublishSkillHubRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + item, err := h.skillService.PublishFromInstance(userID.(int), userRole.(string), instance.ID, skillID, req.TagIDs) + if err != nil { + utils.HandleHubError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill published to hub successfully", item) +} + func (h *InstanceHandler) requireOwnedInstance(c *gin.Context) (*models.Instance, bool) { idStr := c.Param("id") id, err := strconv.Atoi(idStr) diff --git a/backend/internal/handlers/instance_handler_test.go b/backend/internal/handlers/instance_handler_test.go index ab194cb..be8c644 100644 --- a/backend/internal/handlers/instance_handler_test.go +++ b/backend/internal/handlers/instance_handler_test.go @@ -1,6 +1,7 @@ package handlers import ( + "fmt" "net/http" "net/http/httptest" "strings" @@ -8,6 +9,7 @@ import ( "time" "clawreef/internal/models" + "clawreef/internal/repository" "clawreef/internal/services" "github.com/gin-gonic/gin" @@ -323,3 +325,166 @@ func TestBatchDeleteLiteInstancesRejectsProInstance(t *testing.T) { t.Fatalf("response did not explain lite-only rejection: %s", recorder.Body.String()) } } + +type stubSessionUsageObservabilityService struct { + usage *services.InstanceSessionUsageResult + detail *services.InstanceSessionUsageDetail + err error +} + +func (s *stubSessionUsageObservabilityService) ListAuditItems(services.AuditQuery) (*services.AuditListResult, error) { + return nil, nil +} +func (s *stubSessionUsageObservabilityService) GetTraceDetail(string) (*services.AuditTraceDetail, error) { + return nil, nil +} +func (s *stubSessionUsageObservabilityService) GetCostOverview(services.CostQuery) (*services.CostOverview, error) { + return nil, nil +} +func (s *stubSessionUsageObservabilityService) GetInstanceSessionUsage(int, services.InstanceSessionUsageQuery) (*services.InstanceSessionUsageResult, error) { + if s.err != nil { + return nil, s.err + } + return s.usage, nil +} +func (s *stubSessionUsageObservabilityService) GetInstanceSessionUsageDetail(int, string, repository.SessionUsageFilter) (*services.InstanceSessionUsageDetail, error) { + if s.err != nil { + return nil, s.err + } + if s.detail == nil { + return nil, fmt.Errorf("session usage not found") + } + return s.detail, nil +} +func (s *stubSessionUsageObservabilityService) GetInstanceLLMGovernanceStatus(int, map[string]interface{}) (*services.InstanceLLMGovernanceStatus, error) { + return nil, nil +} +func (s *stubSessionUsageObservabilityService) GetLLMGovernanceOverview() (*services.LLMGovernanceOverview, error) { + return nil, nil +} +func (s *stubSessionUsageObservabilityService) GetAdminSessionUsageOverview(services.InstanceSessionUsageOverviewQuery) (*services.InstanceSessionUsageOverview, error) { + return nil, nil +} + +func TestGetInstanceSessionUsageReturns200(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage?page=1&limit=10", nil) + c.Params = gin.Params{{Key: "id", Value: "9"}} + c.Set("userID", 7) + c.Set("userRole", "user") + + handler := &InstanceHandler{ + instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"}, + }}, + aiObservabilityService: &stubSessionUsageObservabilityService{ + usage: &services.InstanceSessionUsageResult{ + Summary: services.InstanceSessionUsageSummary{Currency: "USD"}, + Items: []services.InstanceSessionUsageItem{}, + }, + }, + } + + handler.GetInstanceSessionUsage(c) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusOK, recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), "Instance session usage retrieved successfully") { + t.Fatalf("unexpected body: %s", recorder.Body.String()) + } +} + +func TestGetInstanceSessionUsageInvalidSince(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage?since=bad-timestamp", nil) + c.Params = gin.Params{{Key: "id", Value: "9"}} + c.Set("userID", 7) + c.Set("userRole", "user") + + handler := &InstanceHandler{ + instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"}, + }}, + aiObservabilityService: &stubSessionUsageObservabilityService{}, + } + + handler.GetInstanceSessionUsage(c) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String()) + } +} + +func TestGetInstanceSessionUsageRejectsUntilBeforeSince(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage?since=2026-07-10T00:00:00Z&until=2026-07-01T00:00:00Z", nil) + c.Params = gin.Params{{Key: "id", Value: "9"}} + c.Set("userID", 7) + c.Set("userRole", "user") + + handler := &InstanceHandler{ + instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"}, + }}, + aiObservabilityService: &stubSessionUsageObservabilityService{}, + } + + handler.GetInstanceSessionUsage(c) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String()) + } +} + +func TestGetInstanceSessionUsageDetailRequiresSessionID(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage/detail", nil) + c.Params = gin.Params{{Key: "id", Value: "9"}} + c.Set("userID", 7) + c.Set("userRole", "user") + + handler := &InstanceHandler{ + instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"}, + }}, + aiObservabilityService: &stubSessionUsageObservabilityService{}, + } + + handler.GetInstanceSessionUsageDetail(c) + + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String()) + } +} + +func TestGetInstanceSessionUsageDetailNotFound(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage/detail?session_id=missing", nil) + c.Params = gin.Params{{Key: "id", Value: "9"}} + c.Set("userID", 7) + c.Set("userRole", "user") + + handler := &InstanceHandler{ + instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"}, + }}, + aiObservabilityService: &stubSessionUsageObservabilityService{}, + } + + handler.GetInstanceSessionUsageDetail(c) + + if recorder.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusNotFound, recorder.Body.String()) + } +} diff --git a/backend/internal/handlers/runtime_agent_handler.go b/backend/internal/handlers/runtime_agent_handler.go index 9f46572..79fa46c 100644 --- a/backend/internal/handlers/runtime_agent_handler.go +++ b/backend/internal/handlers/runtime_agent_handler.go @@ -28,6 +28,7 @@ type RuntimeAgentHandler struct { bindingRepo repository.InstanceRuntimeBindingRepository instanceRepo repository.InstanceRepository events runtimeEventPublisher + skillService services.SkillService } type runtimeAgentPodIdentity struct { @@ -89,13 +90,14 @@ type runtimeAgentGatewayReport struct { HealthAt *time.Time `json:"health_at,omitempty"` } -func NewRuntimeAgentHandler(cfg config.RuntimePoolConfig, podRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, instanceRepo repository.InstanceRepository, events runtimeEventPublisher) *RuntimeAgentHandler { +func NewRuntimeAgentHandler(cfg config.RuntimePoolConfig, podRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, instanceRepo repository.InstanceRepository, events runtimeEventPublisher, skillService services.SkillService) *RuntimeAgentHandler { return &RuntimeAgentHandler{ cfg: cfg, podRepo: podRepo, bindingRepo: bindingRepo, instanceRepo: instanceRepo, events: events, + skillService: skillService, } } @@ -351,6 +353,12 @@ func (h *RuntimeAgentHandler) ReportSkills(c *gin.Context) { utils.ValidationError(c, err) return } + if h.skillService != nil { + if err := h.skillService.SyncRuntimeAgentSkillsReport(payload); err != nil { + utils.HandleError(c, err) + return + } + } h.publish(c.Request.Context(), "runtime_agent_skills_reported", payload) utils.Success(c, http.StatusOK, "Runtime agent skills report accepted", nil) } diff --git a/backend/internal/handlers/runtime_agent_handler_test.go b/backend/internal/handlers/runtime_agent_handler_test.go index 629207b..a6b3ae7 100644 --- a/backend/internal/handlers/runtime_agent_handler_test.go +++ b/backend/internal/handlers/runtime_agent_handler_test.go @@ -19,7 +19,7 @@ import ( func TestRuntimeAgentHandlerRejectsInvalidToken(t *testing.T) { gin.SetMode(gin.TestMode) podRepo := &runtimeAgentHandlerPodRepo{} - handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, &runtimeAgentHandlerEvents{}) + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, &runtimeAgentHandlerEvents{}, nil) router := gin.New() router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics) @@ -44,7 +44,7 @@ func TestRuntimeAgentHandlerRegisterUsesConfiguredCapacity(t *testing.T) { handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{ AgentReportToken: "secret", MaxGatewaysPerPod: 33, - }, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events) +}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events, nil) router := gin.New() router.POST("/api/v1/runtime-agent/register", handler.Register) @@ -95,7 +95,7 @@ func TestRuntimeAgentHandlerHeartbeatUsesConfiguredCapacity(t *testing.T) { handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{ AgentReportToken: "secret", MaxGatewaysPerPod: 44, - }, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events) +}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events, nil) router := gin.New() router.POST("/api/v1/runtime-agent/heartbeat", handler.Heartbeat) @@ -134,7 +134,7 @@ func TestRuntimeAgentHandlerMetricsReportUpdatesPodAndPublishesEvent(t *testing. gin.SetMode(gin.TestMode) podRepo := &runtimeAgentHandlerPodRepo{} events := &runtimeAgentHandlerEvents{} - handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events) + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events, nil) router := gin.New() router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics) @@ -187,7 +187,7 @@ func TestRuntimeAgentHandlerGatewayReportOnlyUpdatesCurrentPodBinding(t *testing 12: {InstanceID: 12, RuntimePodID: 9, Generation: 3}, }, } - handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}) + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}, nil) router := gin.New() router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways) @@ -228,7 +228,7 @@ func TestRuntimeAgentHandlerGatewayReportSyncsInstanceRuntimeState(t *testing.T) }, } instanceRepo := &runtimeAgentHandlerInstanceRepo{} - handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, instanceRepo, &runtimeAgentHandlerEvents{}) + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, instanceRepo, &runtimeAgentHandlerEvents{}, nil) router := gin.New() router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways) @@ -274,7 +274,7 @@ func TestRuntimeAgentHandlerGatewayReportDeletesMissingCurrentPodBinding(t *test 12: {InstanceID: 12, RuntimePodID: 9, Generation: 2, State: "error"}, }, } - handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}) + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}, nil) router := gin.New() router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways) @@ -304,7 +304,7 @@ func TestRuntimeAgentHandlerGatewayReportDoesNotDeleteFreshBindingFromFirstEmpty 10: {InstanceID: 10, RuntimePodID: 9, Generation: 3, State: "running", LastHealthAt: &lastHealthAt}, }, } - handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}) + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}, nil) router := gin.New() router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways) diff --git a/backend/internal/handlers/runtime_pool_handler_test.go b/backend/internal/handlers/runtime_pool_handler_test.go index 4d43885..70b2cc4 100644 --- a/backend/internal/handlers/runtime_pool_handler_test.go +++ b/backend/internal/handlers/runtime_pool_handler_test.go @@ -536,3 +536,6 @@ func (c *runtimePoolHandlerAgentClient) DeleteGateway(ctx context.Context, endpo return nil } func (c *runtimePoolHandlerAgentClient) Drain(ctx context.Context, endpoint string) error { return nil } +func (c *runtimePoolHandlerAgentClient) ResyncInstanceSkills(ctx context.Context, endpoint string, instanceID int, mode string) error { + return nil +} diff --git a/backend/internal/handlers/session_usage_query.go b/backend/internal/handlers/session_usage_query.go new file mode 100644 index 0000000..bd84fd5 --- /dev/null +++ b/backend/internal/handlers/session_usage_query.go @@ -0,0 +1,27 @@ +package handlers + +import ( + "fmt" + "strings" + "time" +) + +func parseOptionalRFC3339(value string) (*time.Time, error) { + value = strings.TrimSpace(value) + if value == "" { + return nil, nil + } + parsed, err := time.Parse(time.RFC3339, value) + if err != nil { + return nil, err + } + utc := parsed.UTC() + return &utc, nil +} + +func validateSessionUsageTimeRange(since, until *time.Time) error { + if since != nil && until != nil && !until.After(*since) { + return fmt.Errorf("until must be after since") + } + return nil +} diff --git a/backend/internal/handlers/session_usage_query_test.go b/backend/internal/handlers/session_usage_query_test.go new file mode 100644 index 0000000..29c836a --- /dev/null +++ b/backend/internal/handlers/session_usage_query_test.go @@ -0,0 +1,33 @@ +package handlers + +import ( + "testing" + "time" +) + +func TestValidateSessionUsageTimeRange(t *testing.T) { + since := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + until := time.Date(2026, 7, 2, 0, 0, 0, 0, time.UTC) + if err := validateSessionUsageTimeRange(&since, &until); err != nil { + t.Fatalf("expected valid range, got %v", err) + } + if err := validateSessionUsageTimeRange(&since, &since); err == nil { + t.Fatalf("expected equal since/until to be rejected") + } + if err := validateSessionUsageTimeRange(nil, &until); err != nil { + t.Fatalf("expected open-ended range, got %v", err) + } +} + +func TestParseOptionalRFC3339(t *testing.T) { + if _, err := parseOptionalRFC3339(""); err != nil { + t.Fatalf("empty value should be allowed: %v", err) + } + parsed, err := parseOptionalRFC3339("2026-07-01T00:00:00Z") + if err != nil || parsed == nil { + t.Fatalf("expected parsed timestamp, got %v err=%v", parsed, err) + } + if _, err := parseOptionalRFC3339("not-a-date"); err == nil { + t.Fatalf("expected invalid timestamp error") + } +} diff --git a/backend/internal/handlers/skill_handler.go b/backend/internal/handlers/skill_handler.go index debb7fa..d586da1 100644 --- a/backend/internal/handlers/skill_handler.go +++ b/backend/internal/handlers/skill_handler.go @@ -56,12 +56,13 @@ func (h *SkillHandler) ListAllSkills(c *gin.Context) { func (h *SkillHandler) GetSkill(c *gin.Context) { userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") skillID, err := strconv.Atoi(c.Param("id")) if err != nil { utils.Error(c, http.StatusBadRequest, "invalid skill ID") return } - item, err := h.service.GetSkill(userID.(int), skillID) + item, err := h.service.GetSkill(userID.(int), userRole.(string), skillID) if err != nil { utils.HandleError(c, err) return @@ -91,12 +92,13 @@ func (h *SkillHandler) UpdateSkill(c *gin.Context) { func (h *SkillHandler) DeleteSkill(c *gin.Context) { userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") skillID, err := strconv.Atoi(c.Param("id")) if err != nil { utils.Error(c, http.StatusBadRequest, "invalid skill ID") return } - if err := h.service.DeleteSkill(userID.(int), skillID); err != nil { + if err := h.service.DeleteSkill(userID.(int), userRole.(string), skillID); err != nil { utils.HandleError(c, err) return } @@ -105,12 +107,13 @@ func (h *SkillHandler) DeleteSkill(c *gin.Context) { func (h *SkillHandler) DownloadSkill(c *gin.Context) { userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") skillID, err := strconv.Atoi(c.Param("id")) if err != nil { utils.Error(c, http.StatusBadRequest, "invalid skill ID") return } - content, fileName, err := h.service.DownloadSkill(userID.(int), skillID) + content, fileName, err := h.service.DownloadSkill(userID.(int), userRole.(string), skillID) if err != nil { utils.HandleError(c, err) return @@ -120,25 +123,15 @@ func (h *SkillHandler) DownloadSkill(c *gin.Context) { c.Data(http.StatusOK, "application/zip", content) } -func (h *SkillHandler) DownloadSkillVersionForAgent(c *gin.Context) { - content, fileName, err := h.service.DownloadSkillVersionByExternalID(c.Param("skillVersion")) - if err != nil { - utils.HandleError(c, err) - return - } - c.Header("Content-Type", "application/octet-stream") - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName)) - c.Data(http.StatusOK, "application/octet-stream", content) -} - func (h *SkillHandler) ListVersions(c *gin.Context) { userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") skillID, err := strconv.Atoi(c.Param("id")) if err != nil { utils.Error(c, http.StatusBadRequest, "invalid skill ID") return } - items, err := h.service.ListVersions(userID.(int), skillID) + items, err := h.service.ListVersions(userID.(int), userRole.(string), skillID) if err != nil { utils.HandleError(c, err) return @@ -148,12 +141,13 @@ func (h *SkillHandler) ListVersions(c *gin.Context) { func (h *SkillHandler) ListScanResults(c *gin.Context) { userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") skillID, err := strconv.Atoi(c.Param("id")) if err != nil { utils.Error(c, http.StatusBadRequest, "invalid skill ID") return } - items, err := h.service.ListScanResults(userID.(int), skillID) + items, err := h.service.ListScanResults(userID.(int), userRole.(string), skillID) if err != nil { utils.HandleError(c, err) return @@ -200,9 +194,9 @@ func (h *SkillHandler) AttachSkillToInstance(c *gin.Context) { } userID, _ := c.Get("userID") userRole, _ := c.Get("userRole") - item, err := h.service.AttachSkillToInstanceForActor(instanceID, req.SkillID, userID.(int), fmt.Sprint(userRole)) + item, err := h.service.AttachSkillToInstance(userID.(int), userRole.(string), instanceID, req.SkillID) if err != nil { - utils.HandleError(c, err) + utils.HandleHubError(c, err) return } utils.Success(c, http.StatusCreated, "Skill attached to instance successfully", item) diff --git a/backend/internal/handlers/skill_hub_handler.go b/backend/internal/handlers/skill_hub_handler.go new file mode 100644 index 0000000..cd32084 --- /dev/null +++ b/backend/internal/handlers/skill_hub_handler.go @@ -0,0 +1,253 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type SkillHubHandler struct { + service services.SkillService + instanceService services.InstanceService +} + +func NewSkillHubHandler(service services.SkillService, instanceService services.InstanceService) *SkillHubHandler { + return &SkillHubHandler{service: service, instanceService: instanceService} +} + +func (h *SkillHubHandler) ListCatalog(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + query := services.SkillHubCatalogQuery{ + TagKeys: c.QueryArray("tag_keys"), + Search: strings.TrimSpace(c.Query("q")), + Page: parseIntDefault(c.Query("page"), 1), + PageSize: parseIntDefault(c.Query("page_size"), 20), + } + result, err := h.service.ListHubCatalog(userID.(int), userRole.(string), query) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill hub catalog retrieved successfully", result) +} + +func (h *SkillHubHandler) ListTags(c *gin.Context) { + userRole, _ := c.Get("userRole") + items, err := h.service.ListHubTags(userRole.(string)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill hub tags retrieved successfully", items) +} + +func (h *SkillHubHandler) ListMine(c *gin.Context) { + userID, _ := c.Get("userID") + items, err := h.service.ListMyHubSkills(userID.(int)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "My skill hub items retrieved successfully", items) +} + +func (h *SkillHubHandler) ListAttachable(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + items, err := h.service.ListAttachableSkills(userID.(int), userRole.(string)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Attachable skills retrieved successfully", items) +} + +func (h *SkillHubHandler) GetSkill(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + item, err := h.service.GetSkillHubDetail(userID.(int), userRole.(string), skillID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill hub item retrieved successfully", item) +} + +func (h *SkillHubHandler) PreviewImportSkills(c *gin.Context) { + userID, _ := c.Get("userID") + fileHeader, err := c.FormFile("file") + if err != nil { + utils.Error(c, http.StatusBadRequest, "file is required") + return + } + items, err := h.service.PreviewHubImport(c.Request.Context(), userID.(int), fileHeader) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill import preview generated successfully", items) +} + +func (h *SkillHubHandler) ImportSkills(c *gin.Context) { + userID, _ := c.Get("userID") + fileHeader, err := c.FormFile("file") + if err != nil { + utils.Error(c, http.StatusBadRequest, "file is required") + return + } + var decisions []services.SkillImportDecision + if raw := strings.TrimSpace(c.PostForm("decisions")); raw != "" { + if err := json.Unmarshal([]byte(raw), &decisions); err != nil { + utils.Error(c, http.StatusBadRequest, "invalid decisions payload") + return + } + } + items, err := h.service.ImportHubArchiveWithDecisions(c.Request.Context(), userID.(int), fileHeader, decisions) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Skills imported successfully", items) +} + +func (h *SkillHubHandler) PublishSkill(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + var req services.PublishSkillHubRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + item, err := h.service.PublishToHub(userID.(int), userRole.(string), skillID, req.TagIDs) + if err != nil { + utils.HandleHubError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill published to hub successfully", item) +} + +func (h *SkillHubHandler) UnpublishSkill(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + item, err := h.service.UnpublishFromHub(userID.(int), userRole.(string), skillID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill unpublished from hub successfully", item) +} + +func (h *SkillHubHandler) UpdateTags(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + var req services.UpdateSkillHubTagsRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + item, err := h.service.UpdateHubTags(userID.(int), userRole.(string), skillID, req.TagIDs) + if err != nil { + utils.HandleHubError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill hub tags updated successfully", item) +} + +func (h *SkillHubHandler) DeleteSkill(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + if err := h.service.DeleteSkill(userID.(int), userRole.(string), skillID); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Skill deleted successfully", nil) +} + +func (h *SkillHubHandler) DownloadSkill(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + content, fileName, err := h.service.DownloadSkill(userID.(int), userRole.(string), skillID) + if err != nil { + utils.HandleError(c, err) + return + } + c.Header("Content-Type", "application/zip") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName)) + c.Data(http.StatusOK, "application/zip", content) +} + +func (h *SkillHubHandler) InstallSkill(c *gin.Context) { + userID, _ := c.Get("userID") + userRole, _ := c.Get("userRole") + skillID, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "invalid skill ID") + return + } + var req services.InstallHubSkillRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + item, err := h.service.InstallHubSkill(userID.(int), userRole.(string), skillID, req.InstanceID) + if err != nil { + utils.HandleHubError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Skill installed to instance successfully", item) +} + +func (h *SkillHubHandler) ListAdminSkills(c *gin.Context) { + items, err := h.service.ListAllHubSkillsAdmin() + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Admin skill hub items retrieved successfully", items) +} + +func parseIntDefault(raw string, fallback int) int { + value, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil || value <= 0 { + return fallback + } + return value +} diff --git a/backend/internal/middleware/auth_middleware.go b/backend/internal/middleware/auth_middleware.go index 521dfe2..66fa9e9 100644 --- a/backend/internal/middleware/auth_middleware.go +++ b/backend/internal/middleware/auth_middleware.go @@ -94,6 +94,7 @@ func GatewayAuth(instanceRepo repository.InstanceRepository, bindingRepos ...rep c.Set("userID", instance.UserID) c.Set("instanceID", instance.ID) + c.Set("instanceType", strings.TrimSpace(instance.Type)) c.Set("instanceMode", gatewayInstanceMode(instance.InstanceMode, instance.RuntimeType)) c.Set("runtimeType", strings.TrimSpace(instance.RuntimeType)) if bindingRepo != nil { diff --git a/backend/internal/models/skill.go b/backend/internal/models/skill.go index a17fbdf..9b116b9 100644 --- a/backend/internal/models/skill.go +++ b/backend/internal/models/skill.go @@ -11,6 +11,9 @@ type Skill struct { CurrentVersionID *int `db:"current_version_id" json:"current_version_id,omitempty"` SourceType string `db:"source_type" json:"source_type"` Status string `db:"status" json:"status"` + Visibility string `db:"visibility" json:"visibility"` + PublishedAt *time.Time `db:"published_at" json:"published_at,omitempty"` + PublishedBy *int `db:"published_by" json:"published_by,omitempty"` RiskLevel string `db:"risk_level" json:"risk_level"` LastScannedAt *time.Time `db:"last_scanned_at" json:"last_scanned_at,omitempty"` LastScanResultID *int `db:"last_scan_result_id" json:"last_scan_result_id,omitempty"` @@ -58,6 +61,7 @@ type InstanceSkill struct { SkillVersionID *int `db:"skill_version_id" json:"skill_version_id,omitempty"` SourceType string `db:"source_type" json:"source_type"` InstallPath *string `db:"install_path" json:"install_path,omitempty"` + WorkspaceDir *string `db:"workspace_dir" json:"workspace_dir,omitempty"` ObservedHash *string `db:"observed_hash" json:"observed_hash,omitempty"` Status string `db:"status" json:"status"` LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"` diff --git a/backend/internal/models/skill_hub_tag.go b/backend/internal/models/skill_hub_tag.go new file mode 100644 index 0000000..d1be6c5 --- /dev/null +++ b/backend/internal/models/skill_hub_tag.go @@ -0,0 +1,25 @@ +package models + +import "time" + +type SkillHubTag struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + TagKey string `db:"tag_key" json:"tag_key"` + Name string `db:"name" json:"name"` + Description *string `db:"description" json:"description,omitempty"` + SortOrder int `db:"sort_order" json:"sort_order"` + AdminOnly bool `db:"admin_only" json:"admin_only"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (t SkillHubTag) TableName() string { return "skill_hub_tags" } + +type SkillHubTagAssignment struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + SkillID int `db:"skill_id" json:"skill_id"` + TagID int `db:"tag_id" json:"tag_id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +func (a SkillHubTagAssignment) TableName() string { return "skill_hub_tag_assignments" } diff --git a/backend/internal/models/skill_package_materialize_job.go b/backend/internal/models/skill_package_materialize_job.go new file mode 100644 index 0000000..ad0cb4e --- /dev/null +++ b/backend/internal/models/skill_package_materialize_job.go @@ -0,0 +1,26 @@ +package models + +import "time" + +type SkillPackageMaterializeJob struct { + ID int `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + SkillID int `db:"skill_id" json:"skill_id"` + BlobID int `db:"blob_id" json:"blob_id"` + WorkspaceDir string `db:"workspace_dir" json:"workspace_dir"` + ContentHash string `db:"content_hash" json:"content_hash"` + Status string `db:"status" json:"status"` + AttemptCount int `db:"attempt_count" json:"attempt_count"` + MaxAttempts int `db:"max_attempts" json:"max_attempts"` + LastError *string `db:"last_error" json:"last_error,omitempty"` + IdempotencyKey string `db:"idempotency_key" json:"idempotency_key"` + TriggerSource string `db:"trigger_source" json:"trigger_source"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (j SkillPackageMaterializeJob) TableName() string { + return "skill_package_materialize_jobs" +} diff --git a/backend/internal/repository/audit_event_repository.go b/backend/internal/repository/audit_event_repository.go index 377a364..6741b38 100644 --- a/backend/internal/repository/audit_event_repository.go +++ b/backend/internal/repository/audit_event_repository.go @@ -14,6 +14,7 @@ type AuditEventRepository interface { Create(event *models.AuditEvent) error ListByTraceID(traceID string) ([]models.AuditEvent, error) ListRecent(limit int) ([]models.AuditEvent, error) + CountRecentByInstanceAndEventType(instanceID int, eventType string, since time.Time) (int, error) } type auditEventRepository struct { @@ -92,3 +93,21 @@ func (r *auditEventRepository) ListRecent(limit int) ([]models.AuditEvent, error } return items, nil } + +func (r *auditEventRepository) CountRecentByInstanceAndEventType(instanceID int, eventType string, since time.Time) (int, error) { + row, err := r.sess.SQL().QueryRow(` +SELECT COUNT(*) +FROM audit_events +WHERE instance_id = ? + AND event_type = ? + AND created_at >= ? +`, instanceID, eventType, since) + if err != nil { + return 0, fmt.Errorf("failed to count audit events by instance and type: %w", err) + } + var count int + if err := row.Scan(&count); err != nil { + return 0, fmt.Errorf("failed to scan audit event count: %w", err) + } + return count, nil +} diff --git a/backend/internal/repository/chat_session_repository.go b/backend/internal/repository/chat_session_repository.go index d48b7fc..ef39b6d 100644 --- a/backend/internal/repository/chat_session_repository.go +++ b/backend/internal/repository/chat_session_repository.go @@ -12,6 +12,7 @@ import ( // ChatSessionRepository defines repository operations for chat sessions. type ChatSessionRepository interface { GetBySessionID(sessionID string) (*models.ChatSession, error) + ListByInstanceID(instanceID int) ([]models.ChatSession, error) Save(session *models.ChatSession) error } @@ -60,6 +61,14 @@ func (r *chatSessionRepository) GetBySessionID(sessionID string) (*models.ChatSe return &item, nil } +func (r *chatSessionRepository) ListByInstanceID(instanceID int) ([]models.ChatSession, error) { + var items []models.ChatSession + if err := r.sess.Collection("chat_sessions").Find(db.Cond{"instance_id": instanceID}).OrderBy("-last_activity_at").All(&items); err != nil { + return nil, fmt.Errorf("failed to list chat sessions by instance id: %w", err) + } + return items, nil +} + func (r *chatSessionRepository) Save(session *models.ChatSession) error { now := time.Now() existing, err := r.GetBySessionID(session.SessionID) diff --git a/backend/internal/repository/cost_record_repository.go b/backend/internal/repository/cost_record_repository.go index 51bb713..913c122 100644 --- a/backend/internal/repository/cost_record_repository.go +++ b/backend/internal/repository/cost_record_repository.go @@ -9,12 +9,23 @@ import ( "github.com/upper/db/v4" ) +// InstanceSessionCostAggregate summarizes cost usage for one session on an instance. +type InstanceSessionCostAggregate struct { + SessionID string + PromptTokens int + CompletionTokens int + TotalTokens int + EstimatedCost float64 + Currency string +} + // CostRecordRepository defines repository operations for token and money accounting. type CostRecordRepository interface { Create(record *models.CostRecord) error ListByTraceID(traceID string) ([]models.CostRecord, error) ListByUserID(userID, limit int) ([]models.CostRecord, error) ListRecent(limit int) ([]models.CostRecord, error) + AggregateCostByInstanceSession(instanceID int, filter SessionUsageFilter) ([]InstanceSessionCostAggregate, error) } type costRecordRepository struct { @@ -110,3 +121,50 @@ func (r *costRecordRepository) ListRecent(limit int) ([]models.CostRecord, error } return items, nil } + +func (r *costRecordRepository) AggregateCostByInstanceSession(instanceID int, filter SessionUsageFilter) ([]InstanceSessionCostAggregate, error) { + query := ` +SELECT cr.session_id, + COALESCE(SUM(cr.prompt_tokens), 0), + COALESCE(SUM(cr.completion_tokens), 0), + COALESCE(SUM(cr.total_tokens), 0), + COALESCE(SUM(cr.estimated_cost), 0), + COALESCE(MAX(cr.currency), 'USD') +FROM cost_records cr +INNER JOIN model_invocations mi + ON mi.trace_id = cr.trace_id + AND mi.instance_id = cr.instance_id + AND mi.status != ? +WHERE cr.instance_id = ? + AND cr.session_id IS NOT NULL + AND cr.session_id != ''` + args := []interface{}{models.ModelInvocationStatusBlocked, instanceID} + query, args = appendTimeFilter(query, args, filter, "mi.created_at") + query += ` +GROUP BY cr.session_id` + rows, err := r.sess.SQL().Query(query, args...) + if err != nil { + return nil, fmt.Errorf("failed to aggregate cost records by instance session: %w", err) + } + defer rows.Close() + + items := make([]InstanceSessionCostAggregate, 0) + for rows.Next() { + var item InstanceSessionCostAggregate + if err := rows.Scan( + &item.SessionID, + &item.PromptTokens, + &item.CompletionTokens, + &item.TotalTokens, + &item.EstimatedCost, + &item.Currency, + ); err != nil { + return nil, fmt.Errorf("failed to scan session cost aggregate: %w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to iterate session cost aggregates: %w", err) + } + return items, nil +} diff --git a/backend/internal/repository/instance_command_repository.go b/backend/internal/repository/instance_command_repository.go index 29032f7..b4e2630 100644 --- a/backend/internal/repository/instance_command_repository.go +++ b/backend/internal/repository/instance_command_repository.go @@ -2,6 +2,7 @@ package repository import ( "fmt" + "strings" "time" "clawreef/internal/models" @@ -16,6 +17,7 @@ type InstanceCommandRepository interface { GetByInstanceIdempotencyKey(instanceID int, idempotencyKey string) (*models.InstanceCommand, error) GetNextPendingByInstance(instanceID int) (*models.InstanceCommand, error) ListByInstanceID(instanceID int, limit int) ([]models.InstanceCommand, error) + FindLatestFailedCollectSkillPackage(skillExternalID string) (*models.InstanceCommand, error) } type instanceCommandRepository struct { @@ -95,3 +97,22 @@ func (r *instanceCommandRepository) ListByInstanceID(instanceID int, limit int) } return items, nil } + +func (r *instanceCommandRepository) FindLatestFailedCollectSkillPackage(skillExternalID string) (*models.InstanceCommand, error) { + skillExternalID = strings.TrimSpace(skillExternalID) + if skillExternalID == "" { + return nil, nil + } + pattern := fmt.Sprintf("%%\"skill_id\":\"%s\"%%", skillExternalID) + var item models.InstanceCommand + if err := r.sess.Collection("instance_commands").Find(db.Cond{ + "command_type": "collect_skill_package", + "status": "failed", + }).And("payload_json LIKE ?", pattern).OrderBy("-finished_at", "-id").One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to find failed collect skill package command: %w", err) + } + return &item, nil +} diff --git a/backend/internal/repository/model_invocation_repository.go b/backend/internal/repository/model_invocation_repository.go index a581cd7..50bf107 100644 --- a/backend/internal/repository/model_invocation_repository.go +++ b/backend/internal/repository/model_invocation_repository.go @@ -10,6 +10,17 @@ import ( "github.com/upper/db/v4" ) +// InstanceSessionTokenAggregate summarizes token usage for one session on an instance. +type InstanceSessionTokenAggregate struct { + SessionID string + PromptTokens int + CompletionTokens int + TotalTokens int + InvocationCount int + FirstSeenAt time.Time + LastSeenAt time.Time +} + // ModelInvocationRepository defines repository operations for governed model calls. type ModelInvocationRepository interface { Create(invocation *models.ModelInvocation) error @@ -18,6 +29,9 @@ type ModelInvocationRepository interface { ListBySessionID(sessionID string, limit int) ([]models.ModelInvocation, error) ListByUserID(userID, limit int) ([]models.ModelInvocation, error) ListRecent(limit int) ([]models.ModelInvocation, error) + AggregateByInstanceSession(instanceID int, filter SessionUsageFilter) ([]InstanceSessionTokenAggregate, error) + ListRecentByInstanceSession(instanceID int, sessionID string, limit int, filter SessionUsageFilter) ([]models.ModelInvocation, error) + CountDistinctSessionsByInstance(instanceID int, filter SessionUsageFilter) (int, error) } type modelInvocationRepository struct { @@ -147,6 +161,139 @@ func (r *modelInvocationRepository) ListRecent(limit int) ([]models.ModelInvocat return items, nil } +func (r *modelInvocationRepository) AggregateByInstanceSession(instanceID int, filter SessionUsageFilter) ([]InstanceSessionTokenAggregate, error) { + query := ` +SELECT session_id, + COALESCE(SUM(prompt_tokens), 0), + COALESCE(SUM(completion_tokens), 0), + COALESCE(SUM(total_tokens), 0), + COUNT(*), + MIN(created_at), + MAX(created_at) +FROM model_invocations +WHERE instance_id = ? + AND session_id IS NOT NULL + AND session_id != '' + AND status != ?` + args := []interface{}{instanceID, models.ModelInvocationStatusBlocked} + query, args = appendTimeFilter(query, args, filter, "created_at") + query += ` +GROUP BY session_id` + rows, err := r.sess.SQL().Query(query, args...) + if err != nil { + return nil, fmt.Errorf("failed to aggregate model invocations by instance session: %w", err) + } + defer rows.Close() + + items := make([]InstanceSessionTokenAggregate, 0) + for rows.Next() { + var item InstanceSessionTokenAggregate + if err := rows.Scan( + &item.SessionID, + &item.PromptTokens, + &item.CompletionTokens, + &item.TotalTokens, + &item.InvocationCount, + &item.FirstSeenAt, + &item.LastSeenAt, + ); err != nil { + return nil, fmt.Errorf("failed to scan session token aggregate: %w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to iterate session token aggregates: %w", err) + } + return items, nil +} + +func (r *modelInvocationRepository) ListRecentByInstanceSession(instanceID int, sessionID string, limit int, filter SessionUsageFilter) ([]models.ModelInvocation, error) { + if limit <= 0 { + limit = 20 + } + query := ` +SELECT id, trace_id, session_id, request_id, user_id, instance_id, instance_mode, runtime_type, gateway_id, runtime_pod_id, model_id, provider_type, requested_model, actual_provider_model, traffic_class, request_payload, response_payload, prompt_tokens, completion_tokens, total_tokens, cached_tokens, reasoning_tokens, latency_ms, is_streaming, status, error_message, created_at, completed_at +FROM model_invocations +WHERE instance_id = ? + AND session_id = ? + AND status != ?` + args := []interface{}{instanceID, sessionID, models.ModelInvocationStatusBlocked} + query, args = appendTimeFilter(query, args, filter, "created_at") + query += ` +ORDER BY created_at DESC +LIMIT ?` + args = append(args, limit) + + rows, err := r.sess.SQL().Query(query, args...) + if err != nil { + return nil, fmt.Errorf("failed to list model invocations by instance session: %w", err) + } + defer rows.Close() + + items := make([]models.ModelInvocation, 0, limit) + for rows.Next() { + var item models.ModelInvocation + if err := rows.Scan( + &item.ID, + &item.TraceID, + &item.SessionID, + &item.RequestID, + &item.UserID, + &item.InstanceID, + &item.InstanceMode, + &item.RuntimeType, + &item.GatewayID, + &item.RuntimePodID, + &item.ModelID, + &item.ProviderType, + &item.RequestedModel, + &item.ActualProviderModel, + &item.TrafficClass, + &item.RequestPayload, + &item.ResponsePayload, + &item.PromptTokens, + &item.CompletionTokens, + &item.TotalTokens, + &item.CachedTokens, + &item.ReasoningTokens, + &item.LatencyMs, + &item.IsStreaming, + &item.Status, + &item.ErrorMessage, + &item.CreatedAt, + &item.CompletedAt, + ); err != nil { + return nil, fmt.Errorf("failed to scan model invocation by instance session: %w", err) + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to iterate model invocations by instance session: %w", err) + } + return items, nil +} + +func (r *modelInvocationRepository) CountDistinctSessionsByInstance(instanceID int, filter SessionUsageFilter) (int, error) { + query := ` +SELECT COUNT(DISTINCT session_id) +FROM model_invocations +WHERE instance_id = ? + AND session_id IS NOT NULL + AND session_id != '' + AND status != ?` + args := []interface{}{instanceID, models.ModelInvocationStatusBlocked} + query, args = appendTimeFilter(query, args, filter, "created_at") + row, err := r.sess.SQL().QueryRow(query, args...) + if err != nil { + return 0, fmt.Errorf("failed to count distinct sessions by instance: %w", err) + } + var count int + if err := row.Scan(&count); err != nil { + return 0, fmt.Errorf("failed to scan distinct session count: %w", err) + } + return count, nil +} + func isDuplicateIndexError(err error) bool { if err == nil { return false diff --git a/backend/internal/repository/session_usage_filter.go b/backend/internal/repository/session_usage_filter.go new file mode 100644 index 0000000..8f1a681 --- /dev/null +++ b/backend/internal/repository/session_usage_filter.go @@ -0,0 +1,24 @@ +package repository + +import ( + "fmt" + "time" +) + +// SessionUsageFilter optionally bounds session usage aggregates by timestamp. +type SessionUsageFilter struct { + Since *time.Time + Until *time.Time +} + +func appendTimeFilter(query string, args []interface{}, filter SessionUsageFilter, column string) (string, []interface{}) { + if filter.Since != nil { + query += fmt.Sprintf(" AND %s >= ?", column) + args = append(args, *filter.Since) + } + if filter.Until != nil { + query += fmt.Sprintf(" AND %s < ?", column) + args = append(args, *filter.Until) + } + return query, args +} diff --git a/backend/internal/repository/session_usage_filter_test.go b/backend/internal/repository/session_usage_filter_test.go new file mode 100644 index 0000000..12bca12 --- /dev/null +++ b/backend/internal/repository/session_usage_filter_test.go @@ -0,0 +1,39 @@ +package repository + +import ( + "strings" + "testing" + "time" +) + +func TestAppendTimeFilterAddsSinceAndUntilClauses(t *testing.T) { + since := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + until := time.Date(2026, 7, 2, 0, 0, 0, 0, time.UTC) + query := "SELECT 1 FROM model_invocations WHERE instance_id = ?" + args := []interface{}{9} + + query, args = appendTimeFilter(query, args, SessionUsageFilter{ + Since: &since, + Until: &until, + }, "created_at") + + if !strings.Contains(query, "created_at >= ?") || !strings.Contains(query, "created_at < ?") { + t.Fatalf("expected created_at bounds in query, got %q", query) + } + if len(args) != 3 { + t.Fatalf("expected 3 args, got %d (%v)", len(args), args) + } + if args[1] != since || args[2] != until { + t.Fatalf("unexpected bound args: %+v", args[1:]) + } +} + +func TestAppendTimeFilterEmptyFilterLeavesQueryUnchanged(t *testing.T) { + query := "SELECT 1 FROM cost_records WHERE instance_id = ?" + args := []interface{}{9} + + updatedQuery, updatedArgs := appendTimeFilter(query, args, SessionUsageFilter{}, "recorded_at") + if updatedQuery != query || len(updatedArgs) != 1 { + t.Fatalf("expected unchanged query/args, got query=%q args=%v", updatedQuery, updatedArgs) + } +} diff --git a/backend/internal/repository/skill_package_materialize_job_repository.go b/backend/internal/repository/skill_package_materialize_job_repository.go new file mode 100644 index 0000000..4366b53 --- /dev/null +++ b/backend/internal/repository/skill_package_materialize_job_repository.go @@ -0,0 +1,335 @@ +package repository + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type SkillPackageMaterializeBackfillCandidate struct { + InstanceID int + SkillID int + BlobID int + WorkspaceDir string + ContentHash string +} + +type SkillPackageMaterializeJobRepository interface { + Create(job *models.SkillPackageMaterializeJob) error + GetByID(id int) (*models.SkillPackageMaterializeJob, error) + GetByIdempotencyKey(key string) (*models.SkillPackageMaterializeJob, error) + ClaimNextPending(ctx context.Context, limit int) ([]models.SkillPackageMaterializeJob, error) + MarkSucceeded(id int) error + MarkFailed(id int, errMsg string, retryable bool) error + MarkRunning(id int) error + ReleaseToPending(id int) error + ResetForRetry(skillID int) error + RequeueExisting(id, blobID int, contentHash, workspaceDir string) error + FindLatestBySkillID(skillID int) (*models.SkillPackageMaterializeJob, error) + CountPendingByInstance(instanceID int) (int, error) + ListBackfillCandidates(limit int) ([]SkillPackageMaterializeBackfillCandidate, error) +} + +type skillPackageMaterializeJobRepository struct { + sess db.Session +} + +func NewSkillPackageMaterializeJobRepository(sess db.Session) SkillPackageMaterializeJobRepository { + return &skillPackageMaterializeJobRepository{sess: sess} +} + +func (r *skillPackageMaterializeJobRepository) Create(job *models.SkillPackageMaterializeJob) error { + existing, err := r.GetByIdempotencyKey(job.IdempotencyKey) + if err != nil { + return err + } + if existing != nil { + *job = *existing + return nil + } + if strings.TrimSpace(job.Status) == "" { + job.Status = "pending" + } + if job.MaxAttempts <= 0 { + job.MaxAttempts = 5 + } + if strings.TrimSpace(job.TriggerSource) == "" { + job.TriggerSource = "sync" + } + ensureTimestamps(&job.CreatedAt, &job.UpdatedAt) + res, err := r.sess.Collection("skill_package_materialize_jobs").Insert(job) + if err != nil { + if isDuplicateEntryError(err) { + existing, findErr := r.GetByIdempotencyKey(job.IdempotencyKey) + if findErr != nil { + return findErr + } + if existing != nil { + *job = *existing + return nil + } + } + return fmt.Errorf("failed to create skill package materialize job: %w", err) + } + if id, ok := res.ID().(int64); ok { + job.ID = int(id) + } + return nil +} + +func (r *skillPackageMaterializeJobRepository) GetByID(id int) (*models.SkillPackageMaterializeJob, error) { + var item models.SkillPackageMaterializeJob + if err := r.sess.Collection("skill_package_materialize_jobs").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill package materialize job: %w", err) + } + return &item, nil +} + +func (r *skillPackageMaterializeJobRepository) GetByIdempotencyKey(key string) (*models.SkillPackageMaterializeJob, error) { + key = strings.TrimSpace(key) + if key == "" { + return nil, nil + } + var item models.SkillPackageMaterializeJob + if err := r.sess.Collection("skill_package_materialize_jobs").Find(db.Cond{"idempotency_key": key}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill package materialize job by idempotency key: %w", err) + } + return &item, nil +} + +func (r *skillPackageMaterializeJobRepository) ClaimNextPending(ctx context.Context, limit int) ([]models.SkillPackageMaterializeJob, error) { + if limit <= 0 { + limit = 1 + } + var ids []int + iter := r.sess.SQL().IteratorContext(ctx, ` + SELECT id FROM skill_package_materialize_jobs + WHERE status = 'pending' + ORDER BY created_at ASC, id ASC + LIMIT ?`, limit) + for iter.Next() { + var id int + if err := iter.Scan(&id); err != nil { + iter.Close() + return nil, fmt.Errorf("failed to scan pending materialize job id: %w", err) + } + ids = append(ids, id) + } + if err := iter.Err(); err != nil { + iter.Close() + return nil, fmt.Errorf("failed to list pending materialize jobs: %w", err) + } + iter.Close() + + now := time.Now().UTC() + claimed := make([]models.SkillPackageMaterializeJob, 0, len(ids)) + for _, id := range ids { + res, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE skill_package_materialize_jobs + SET status = 'running', + attempt_count = attempt_count + 1, + started_at = ?, + updated_at = ? + WHERE id = ? AND status = 'pending'`, now, now, id) + if err != nil { + return nil, fmt.Errorf("failed to claim materialize job %d: %w", id, err) + } + affected, err := res.RowsAffected() + if err != nil { + return nil, fmt.Errorf("failed to read claim rows affected for job %d: %w", id, err) + } + if affected == 0 { + continue + } + item, err := r.GetByID(id) + if err != nil { + return nil, err + } + if item != nil { + claimed = append(claimed, *item) + } + } + return claimed, nil +} + +func (r *skillPackageMaterializeJobRepository) MarkSucceeded(id int) error { + now := time.Now().UTC() + _, err := r.sess.SQL().Exec(` + UPDATE skill_package_materialize_jobs + SET status = 'succeeded', + finished_at = ?, + last_error = NULL, + updated_at = ? + WHERE id = ?`, now, now, id) + if err != nil { + return fmt.Errorf("failed to mark materialize job succeeded: %w", err) + } + return nil +} + +func (r *skillPackageMaterializeJobRepository) MarkRunning(id int) error { + now := time.Now().UTC() + _, err := r.sess.SQL().Exec(` + UPDATE skill_package_materialize_jobs + SET status = 'running', + attempt_count = attempt_count + 1, + started_at = ?, + updated_at = ? + WHERE id = ? AND status = 'pending'`, now, now, id) + if err != nil { + return fmt.Errorf("failed to mark materialize job running: %w", err) + } + return nil +} + +func (r *skillPackageMaterializeJobRepository) MarkFailed(id int, errMsg string, retryable bool) error { + job, err := r.GetByID(id) + if err != nil { + return err + } + if job == nil { + return fmt.Errorf("materialize job not found") + } + now := time.Now().UTC() + status := "failed" + if retryable && job.AttemptCount < job.MaxAttempts { + status = "pending" + } + trimmed := strings.TrimSpace(errMsg) + var lastError *string + if trimmed != "" { + lastError = &trimmed + } + update := map[string]interface{}{ + "status": status, + "last_error": lastError, + "updated_at": now, + } + if status == "failed" { + update["finished_at"] = now + } + if err := r.sess.Collection("skill_package_materialize_jobs").Find(db.Cond{"id": id}).Update(update); err != nil { + return fmt.Errorf("failed to mark materialize job failed: %w", err) + } + return nil +} + +func (r *skillPackageMaterializeJobRepository) ReleaseToPending(id int) error { + now := time.Now().UTC() + _, err := r.sess.SQL().Exec(` + UPDATE skill_package_materialize_jobs + SET status = 'pending', + started_at = NULL, + updated_at = ? + WHERE id = ? AND status = 'running'`, now, id) + if err != nil { + return fmt.Errorf("failed to release materialize job to pending: %w", err) + } + return nil +} + +func (r *skillPackageMaterializeJobRepository) ResetForRetry(skillID int) error { + job, err := r.FindLatestBySkillID(skillID) + if err != nil { + return err + } + if job == nil { + return fmt.Errorf("materialize job not found") + } + return r.RequeueExisting(job.ID, job.BlobID, job.ContentHash, job.WorkspaceDir) +} + +func (r *skillPackageMaterializeJobRepository) RequeueExisting(id, blobID int, contentHash, workspaceDir string) error { + now := time.Now().UTC() + _, err := r.sess.SQL().Exec(` + UPDATE skill_package_materialize_jobs + SET status = 'pending', + last_error = NULL, + finished_at = NULL, + started_at = NULL, + blob_id = ?, + content_hash = ?, + workspace_dir = ?, + updated_at = ? + WHERE id = ?`, blobID, strings.TrimSpace(contentHash), sanitizeMaterializeWorkspaceDir(workspaceDir), now, id) + if err != nil { + return fmt.Errorf("failed to requeue materialize job: %w", err) + } + return nil +} + +func sanitizeMaterializeWorkspaceDir(value string) string { + return strings.TrimSpace(value) +} + +func (r *skillPackageMaterializeJobRepository) FindLatestBySkillID(skillID int) (*models.SkillPackageMaterializeJob, error) { + var item models.SkillPackageMaterializeJob + if err := r.sess.Collection("skill_package_materialize_jobs").Find(db.Cond{"skill_id": skillID}).OrderBy("-created_at", "-id").One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to find latest materialize job: %w", err) + } + return &item, nil +} + +func (r *skillPackageMaterializeJobRepository) CountPendingByInstance(instanceID int) (int, error) { + row, err := r.sess.SQL().QueryRow(` + SELECT COUNT(*) FROM skill_package_materialize_jobs + WHERE instance_id = ? AND status IN ('pending', 'running')`, instanceID) + if err != nil { + return 0, fmt.Errorf("failed to count pending materialize jobs: %w", err) + } + var count int + if err := row.Scan(&count); err != nil { + return 0, fmt.Errorf("failed to scan pending materialize job count: %w", err) + } + return count, nil +} + +func (r *skillPackageMaterializeJobRepository) ListBackfillCandidates(limit int) ([]SkillPackageMaterializeBackfillCandidate, error) { + if limit <= 0 { + limit = 500 + } + iter := r.sess.SQL().Iterator(` + SELECT isk.instance_id, isk.skill_id, sv.blob_id, isk.workspace_dir, sb.content_hash + FROM instance_skills isk + JOIN instances i ON i.id = isk.instance_id + JOIN skills s ON s.id = isk.skill_id + JOIN skill_versions sv ON sv.id = s.current_version_id + JOIN skill_blobs sb ON sb.id = sv.blob_id + WHERE isk.status = 'active' + AND (LOWER(TRIM(i.instance_mode)) = 'lite' OR LOWER(TRIM(i.runtime_type)) = 'gateway') + AND TRIM(sb.object_key) = '' + AND isk.workspace_dir IS NOT NULL + AND TRIM(isk.workspace_dir) <> '' + ORDER BY isk.updated_at ASC + LIMIT ?`, limit) + defer iter.Close() + + result := make([]SkillPackageMaterializeBackfillCandidate, 0) + for iter.Next() { + var item SkillPackageMaterializeBackfillCandidate + if err := iter.Scan(&item.InstanceID, &item.SkillID, &item.BlobID, &item.WorkspaceDir, &item.ContentHash); err != nil { + return nil, fmt.Errorf("failed to scan backfill candidate: %w", err) + } + result = append(result, item) + } + if err := iter.Err(); err != nil && !errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("failed to list backfill candidates: %w", err) + } + return result, nil +} diff --git a/backend/internal/repository/skill_repository.go b/backend/internal/repository/skill_repository.go index 40e36f9..6ec5ba0 100644 --- a/backend/internal/repository/skill_repository.go +++ b/backend/internal/repository/skill_repository.go @@ -27,7 +27,9 @@ type SkillRepository interface { GetVersionBySkillAndBlob(skillID, blobID int) (*models.SkillVersion, error) GetLatestVersionBySkillID(skillID int) (*models.SkillVersion, error) CreateVersion(version *models.SkillVersion) error + UpdateVersion(version *models.SkillVersion) error ListInstanceSkills(instanceID int) ([]models.InstanceSkill, error) + ListActiveInstanceSkillsBySkillID(skillID int) ([]models.InstanceSkill, error) GetInstanceSkill(instanceID, skillID int) (*models.InstanceSkill, error) UpsertInstanceSkill(item *models.InstanceSkill) error MarkInstanceSkillRemoved(instanceID int, skillID int, observedAt time.Time) error @@ -39,6 +41,12 @@ type SkillRepository interface { ListScanResultsByBlobID(blobID int) ([]models.SkillScanResult, error) GetLatestScanResultByBlobID(blobID int) (*models.SkillScanResult, error) GetLatestScanResultBySkillID(skillID int) (*models.SkillScanResult, error) + ListHubTags(includeAdminOnly bool) ([]models.SkillHubTag, error) + GetHubTagByID(id int) (*models.SkillHubTag, error) + ListHubTagsBySkillID(skillID int) ([]models.SkillHubTag, error) + ReplaceSkillTagAssignments(skillID int, tagIDs []int) error + ListPublicHubSkills() ([]models.Skill, error) + ListSkillsForHubAdmin() ([]models.Skill, error) } type skillRepository struct{ sess db.Session } @@ -74,7 +82,7 @@ func (r *skillRepository) GetSkillByID(id int) (*models.Skill, error) { func (r *skillRepository) GetSkillByUserKey(userID int, skillKey string) (*models.Skill, error) { var item models.Skill - if err := r.sess.Collection("skills").Find(db.Cond{"user_id": userID, "skill_key": skillKey}).One(&item); err != nil { + if err := r.sess.Collection("skills").Find(db.Cond{"user_id": userID, "skill_key": skillKey, "status": "active"}).One(&item); err != nil { if err == db.ErrNoMoreRows { return nil, nil } @@ -84,6 +92,9 @@ func (r *skillRepository) GetSkillByUserKey(userID int, skillKey string) (*model } func (r *skillRepository) CreateSkill(skill *models.Skill) error { + if strings.TrimSpace(skill.Visibility) == "" { + skill.Visibility = "private" + } ensureTimestamps(&skill.CreatedAt, &skill.UpdatedAt) res, err := r.sess.Collection("skills").Insert(skill) if err != nil { @@ -219,6 +230,16 @@ func (r *skillRepository) CreateVersion(version *models.SkillVersion) error { return nil } +func (r *skillRepository) UpdateVersion(version *models.SkillVersion) error { + if version.UpdatedAt.IsZero() { + version.UpdatedAt = time.Now().UTC() + } + if err := r.sess.Collection("skill_versions").Find(db.Cond{"id": version.ID}).Update(version); err != nil { + return fmt.Errorf("failed to update skill version: %w", err) + } + return nil +} + func (r *skillRepository) ListInstanceSkills(instanceID int) ([]models.InstanceSkill, error) { var items []models.InstanceSkill if err := r.sess.Collection("instance_skills").Find(db.Cond{"instance_id": instanceID}).OrderBy("-updated_at", "-id").All(&items); err != nil { @@ -227,6 +248,17 @@ func (r *skillRepository) ListInstanceSkills(instanceID int) ([]models.InstanceS return items, nil } +func (r *skillRepository) ListActiveInstanceSkillsBySkillID(skillID int) ([]models.InstanceSkill, error) { + var items []models.InstanceSkill + if err := r.sess.Collection("instance_skills").Find(db.Cond{ + "skill_id": skillID, + "status NOT IN": []string{"removed", "missing"}, + }).OrderBy("-updated_at", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list active instance skills by skill id: %w", err) + } + return items, nil +} + func (r *skillRepository) GetInstanceSkill(instanceID, skillID int) (*models.InstanceSkill, error) { var item models.InstanceSkill if err := r.sess.Collection("instance_skills").Find(db.Cond{"instance_id": instanceID, "skill_id": skillID}).One(&item); err != nil { @@ -262,6 +294,9 @@ func (r *skillRepository) UpsertInstanceSkill(item *models.InstanceSkill) error if item.UpdatedAt.IsZero() { item.UpdatedAt = time.Now().UTC() } + if existing.SourceType == "injected_by_clawmanager" && item.SourceType == "discovered_in_instance" { + item.SourceType = existing.SourceType + } if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": existing.ID}).Update(item); err != nil { return fmt.Errorf("failed to update instance skill after duplicate insert: %w", err) } @@ -277,6 +312,9 @@ func (r *skillRepository) UpsertInstanceSkill(item *models.InstanceSkill) error if item.UpdatedAt.IsZero() { item.UpdatedAt = time.Now().UTC() } + if existing.SourceType == "injected_by_clawmanager" && item.SourceType == "discovered_in_instance" { + item.SourceType = existing.SourceType + } if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": existing.ID}).Update(item); err != nil { return fmt.Errorf("failed to update instance skill: %w", err) } @@ -407,11 +445,9 @@ func workspaceDeleteTargetsSkillKey(deletedPath string, skillKey string) bool { if len(segments) <= 2 { return true } - for i, segment := range segments { - if segment == "skills" && i+1 < len(segments) && segments[i+1] == key { - return true - } - if segment == ".openclaw" || segment == "openclaw" { + for _, segment := range segments { + switch segment { + case "skills", ".hermes", "hermes", ".openclaw", "openclaw": return true } } @@ -419,7 +455,8 @@ func workspaceDeleteTargetsSkillKey(deletedPath string, skillKey string) bool { } func isRemovedInstanceSkillRecord(item models.InstanceSkill) bool { - return strings.EqualFold(strings.TrimSpace(item.Status), "removed") || item.RemovedAt != nil + status := strings.ToLower(strings.TrimSpace(item.Status)) + return status == "removed" || status == "missing" } func (r *skillRepository) MarkMissingInstanceSkills(instanceID int, activeSkillIDs []int, observedAt time.Time) error { @@ -432,11 +469,14 @@ func (r *skillRepository) MarkMissingInstanceSkills(instanceID int, activeSkillI return fmt.Errorf("failed to list stale instance skills: %w", err) } for _, item := range items { - item.Status = "removed" + if strings.EqualFold(strings.TrimSpace(item.Status), "removed") { + continue + } + item.Status = "missing" item.RemovedAt = &observedAt item.UpdatedAt = observedAt if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": item.ID}).Update(item); err != nil { - return fmt.Errorf("failed to mark instance skill removed: %w", err) + return fmt.Errorf("failed to mark instance skill missing: %w", err) } } return nil @@ -491,3 +531,79 @@ func (r *skillRepository) GetLatestScanResultBySkillID(skillID int) (*models.Ski } return r.GetScanResultByID(*skill.LastScanResultID) } + +func (r *skillRepository) ListHubTags(includeAdminOnly bool) ([]models.SkillHubTag, error) { + var items []models.SkillHubTag + query := r.sess.Collection("skill_hub_tags").Find() + if !includeAdminOnly { + query = query.And(db.Cond{"admin_only": false}) + } + if err := query.OrderBy("sort_order", "id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list skill hub tags: %w", err) + } + return items, nil +} + +func (r *skillRepository) GetHubTagByID(id int) (*models.SkillHubTag, error) { + var item models.SkillHubTag + if err := r.sess.Collection("skill_hub_tags").Find(db.Cond{"id": id}).One(&item); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get skill hub tag: %w", err) + } + return &item, nil +} + +func (r *skillRepository) ListHubTagsBySkillID(skillID int) ([]models.SkillHubTag, error) { + var assignments []models.SkillHubTagAssignment + if err := r.sess.Collection("skill_hub_tag_assignments").Find(db.Cond{"skill_id": skillID}).All(&assignments); err != nil { + return nil, fmt.Errorf("failed to list skill hub tag assignments: %w", err) + } + if len(assignments) == 0 { + return []models.SkillHubTag{}, nil + } + tagIDs := make([]interface{}, 0, len(assignments)) + for _, item := range assignments { + tagIDs = append(tagIDs, item.TagID) + } + var tags []models.SkillHubTag + if err := r.sess.Collection("skill_hub_tags").Find(db.Cond{"id IN": tagIDs}).OrderBy("sort_order", "id").All(&tags); err != nil { + return nil, fmt.Errorf("failed to list skill hub tags by skill id: %w", err) + } + return tags, nil +} + +func (r *skillRepository) ReplaceSkillTagAssignments(skillID int, tagIDs []int) error { + if err := r.sess.Collection("skill_hub_tag_assignments").Find(db.Cond{"skill_id": skillID}).Delete(); err != nil { + return fmt.Errorf("failed to clear skill hub tag assignments: %w", err) + } + for _, tagID := range tagIDs { + assignment := &models.SkillHubTagAssignment{SkillID: skillID, TagID: tagID} + ensureTimestamps(&assignment.CreatedAt, nil) + if _, err := r.sess.Collection("skill_hub_tag_assignments").Insert(assignment); err != nil { + return fmt.Errorf("failed to create skill hub tag assignment: %w", err) + } + } + return nil +} + +func (r *skillRepository) ListPublicHubSkills() ([]models.Skill, error) { + var items []models.Skill + if err := r.sess.Collection("skills").Find(db.Cond{ + "visibility": "public", + "source_type": "uploaded", + "status": "active", + }).OrderBy("-published_at", "-updated_at", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list public hub skills: %w", err) + } + return items, nil +} + +func (r *skillRepository) ListSkillsForHubAdmin() ([]models.Skill, error) { + var items []models.Skill + if err := r.sess.Collection("skills").Find(db.Cond{"source_type": "uploaded"}).OrderBy("-updated_at", "-id").All(&items); err != nil { + return nil, fmt.Errorf("failed to list skills for hub admin: %w", err) + } + return items, nil +} diff --git a/backend/internal/repository/skill_repository_test.go b/backend/internal/repository/skill_repository_test.go new file mode 100644 index 0000000..3b4c43b --- /dev/null +++ b/backend/internal/repository/skill_repository_test.go @@ -0,0 +1,24 @@ +package repository + +import "testing" + +func TestWorkspaceDeleteTargetsSkillKeyHermesNestedPath(t *testing.T) { + path := "home/.hermes/skills/productivity/my-skill" + if !workspaceDeleteTargetsSkillKey(path, "my-skill") { + t.Fatalf("expected nested hermes skill path to match") + } +} + +func TestWorkspaceDeleteTargetsSkillKeyOpenClawFlatPath(t *testing.T) { + path := "home/.openclaw/workspace/skills/paper-ranker" + if !workspaceDeleteTargetsSkillKey(path, "paper-ranker") { + t.Fatalf("expected openclaw flat skill path to match") + } +} + +func TestWorkspaceDeleteTargetsSkillKeyRejectsMismatchedLeaf(t *testing.T) { + path := "home/.hermes/skills/productivity/other-skill" + if workspaceDeleteTargetsSkillKey(path, "my-skill") { + t.Fatalf("expected mismatched leaf to be rejected") + } +} diff --git a/backend/internal/services/ai_observability_governance_overview_test.go b/backend/internal/services/ai_observability_governance_overview_test.go new file mode 100644 index 0000000..2ee8647 --- /dev/null +++ b/backend/internal/services/ai_observability_governance_overview_test.go @@ -0,0 +1,109 @@ +package services + +import ( + "context" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +type stubGovernanceInstanceRepo struct { + instances []models.Instance +} + +func (s *stubGovernanceInstanceRepo) Create(*models.Instance) error { panic("not used") } +func (s *stubGovernanceInstanceRepo) GetByID(int) (*models.Instance, error) { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) GetByAccessToken(string) (*models.Instance, error) { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) GetAll(int, int) ([]models.Instance, error) { panic("not used") } +func (s *stubGovernanceInstanceRepo) CountAll() (int, error) { panic("not used") } +func (s *stubGovernanceInstanceRepo) GetByUserID(int, int, int) ([]models.Instance, error) { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) CountByUserID(int) (int, error) { panic("not used") } +func (s *stubGovernanceInstanceRepo) CountActiveByMode(context.Context, string) (int, error) { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) ExistsByUserIDAndName(int, string) (bool, error) { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) GetAllRunning() ([]models.Instance, error) { + return s.instances, nil +} +func (s *stubGovernanceInstanceRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) UpdateRuntimeState(context.Context, int, string, int, *string) error { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) SetWorkspacePath(context.Context, int, string) error { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) UpdateWorkspaceUsage(context.Context, int, int64) error { + panic("not used") +} +func (s *stubGovernanceInstanceRepo) Update(*models.Instance) error { panic("not used") } +func (s *stubGovernanceInstanceRepo) Delete(int) error { panic("not used") } + +type stubGovernanceRuntimeStatusRepo struct { + external map[int]bool +} + +func (s *stubGovernanceRuntimeStatusRepo) GetByInstanceID(instanceID int) (*models.InstanceRuntimeStatus, error) { + if s.external[instanceID] { + raw := `{"llm_config_status":"external"}` + return &models.InstanceRuntimeStatus{SystemInfoJSON: &raw}, nil + } + return nil, nil +} +func (s *stubGovernanceRuntimeStatusRepo) Create(*models.InstanceRuntimeStatus) error { panic("not used") } +func (s *stubGovernanceRuntimeStatusRepo) Update(*models.InstanceRuntimeStatus) error { panic("not used") } + +func TestGetLLMGovernanceOverviewSummarizesManagedRuntimeInstances(t *testing.T) { + now := time.Now().UTC() + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + {SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now, FirstSeenAt: now}, + }, + }, + costRepo: &stubSessionUsageCostRepo{}, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + auditRepo: &stubSessionUsageAuditRepo{}, + instanceRepo: &stubGovernanceInstanceRepo{ + instances: []models.Instance{ + {ID: 1, UserID: 9, Name: "oc-1", Type: "openclaw", Status: "running"}, + {ID: 2, UserID: 9, Name: "oc-2", Type: "openclaw", Status: "running"}, + {ID: 3, UserID: 9, Name: "ubuntu", Type: "ubuntu", Status: "running"}, + }, + }, + runtimeStatusRepo: &stubGovernanceRuntimeStatusRepo{ + external: map[int]bool{2: true}, + }, + } + + overview, err := service.GetLLMGovernanceOverview() + if err != nil { + t.Fatalf("GetLLMGovernanceOverview failed: %v", err) + } + if overview.TotalManagedInstances != 2 { + t.Fatalf("expected 2 managed instances, got %d", overview.TotalManagedInstances) + } + if overview.ExternalConfigCount != 1 { + t.Fatalf("expected 1 external config instance, got %d", overview.ExternalConfigCount) + } + if len(overview.Items) != 2 { + t.Fatalf("expected 2 overview items, got %d", len(overview.Items)) + } +} diff --git a/backend/internal/services/ai_observability_service.go b/backend/internal/services/ai_observability_service.go index 36ce777..c9e429a 100644 --- a/backend/internal/services/ai_observability_service.go +++ b/backend/internal/services/ai_observability_service.go @@ -4,11 +4,13 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "strings" "time" "clawreef/internal/models" "clawreef/internal/repository" + "clawreef/internal/utils" ) // AuditQuery contains query options for AI audit list views. @@ -160,11 +162,152 @@ type CostRecordView struct { RecordedAt time.Time `json:"recorded_at"` } +// InstanceSessionUsageQuery contains query options for instance session usage views. +type InstanceSessionUsageQuery struct { + Page int + Limit int + Search string + Since *time.Time + Until *time.Time +} + +// InstanceSessionUsageSummary summarizes token usage across all sessions on an instance. +type InstanceSessionUsageSummary struct { + TotalPromptTokens int `json:"total_prompt_tokens"` + TotalCompletionTokens int `json:"total_completion_tokens"` + TotalTokens int `json:"total_tokens"` + TotalEstimatedCost float64 `json:"total_estimated_cost"` + Currency string `json:"currency"` + SessionCount int `json:"session_count"` +} + +// InstanceSessionUsageCompliance reports session attribution quality for an instance. +type InstanceSessionUsageCompliance struct { + FallbackSessionCount int `json:"fallback_session_count"` + HasFallbackSessions bool `json:"has_fallback_sessions"` + RecentFallbackAuditCount int `json:"recent_fallback_audit_count"` +} + +// InstanceSessionUsageItem is one session row for instance usage reporting. +type InstanceSessionUsageItem struct { + SessionID string `json:"session_id"` + SessionKey string `json:"session_key"` + Title *string `json:"title,omitempty"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + EstimatedCost float64 `json:"estimated_cost"` + Currency string `json:"currency"` + InvocationCount int `json:"invocation_count"` + FirstSeenAt time.Time `json:"first_seen_at"` + LastSeenAt time.Time `json:"last_seen_at"` +} + +// InstanceSessionUsageResult is the paginated instance session usage response. +type InstanceSessionUsageResult struct { + Summary InstanceSessionUsageSummary `json:"summary"` + Compliance InstanceSessionUsageCompliance `json:"compliance"` + Items []InstanceSessionUsageItem `json:"items"` + Total int `json:"total"` + Page int `json:"page"` + Limit int `json:"limit"` +} + +// InstanceSessionTrace is a recent trace row within one session. +type InstanceSessionTrace struct { + TraceID string `json:"trace_id"` + RequestedModel string `json:"requested_model"` + Status string `json:"status"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + CreatedAt time.Time `json:"created_at"` +} + +// InstanceSessionUsageDetail is the drill-down payload for one session on an instance. +type InstanceSessionUsageDetail struct { + SessionID string `json:"session_id"` + SessionKey string `json:"session_key"` + Title *string `json:"title,omitempty"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + EstimatedCost float64 `json:"estimated_cost"` + Currency string `json:"currency"` + InvocationCount int `json:"invocation_count"` + FirstSeenAt time.Time `json:"first_seen_at"` + LastSeenAt time.Time `json:"last_seen_at"` + ModelBreakdown []CostBreakdownItem `json:"model_breakdown"` + RecentTraces []InstanceSessionTrace `json:"recent_traces"` +} + +// InstanceLLMGovernanceStatus summarizes LLM governance health for one instance. +type InstanceLLMGovernanceStatus struct { + ConfigStatus string `json:"config_status"` + SessionFallbackRate float64 `json:"session_fallback_rate"` + RecentEgressBlockCount int `json:"recent_egress_block_count"` + IsCompliant bool `json:"is_compliant"` +} + +// LLMGovernanceOverviewItem summarizes governance for one managed runtime instance. +type LLMGovernanceOverviewItem struct { + InstanceID int `json:"instance_id"` + InstanceName string `json:"instance_name"` + InstanceType string `json:"instance_type"` + UserID int `json:"user_id"` + ConfigStatus string `json:"config_status"` + SessionFallbackRate float64 `json:"session_fallback_rate"` + RecentEgressBlockCount int `json:"recent_egress_block_count"` + IsCompliant bool `json:"is_compliant"` +} + +// LLMGovernanceOverview aggregates governance signals across managed runtime instances. +type LLMGovernanceOverview struct { + TotalManagedInstances int `json:"total_managed_instances"` + NonCompliantCount int `json:"non_compliant_count"` + ExternalConfigCount int `json:"external_config_count"` + HighFallbackCount int `json:"high_fallback_count"` + Items []LLMGovernanceOverviewItem `json:"items"` +} + +// InstanceSessionUsageOverviewQuery contains query options for admin session usage overview. +type InstanceSessionUsageOverviewQuery struct { + Page int + Limit int + Search string + Since *time.Time + Until *time.Time +} + +// InstanceSessionUsageOverviewItem summarizes session usage for one managed runtime instance. +type InstanceSessionUsageOverviewItem struct { + InstanceID int `json:"instance_id"` + InstanceName string `json:"instance_name"` + InstanceType string `json:"instance_type"` + UserID int `json:"user_id"` + Summary InstanceSessionUsageSummary `json:"summary"` + Compliance InstanceSessionUsageCompliance `json:"compliance"` +} + +// InstanceSessionUsageOverview aggregates session usage across managed runtime instances. +type InstanceSessionUsageOverview struct { + Summary InstanceSessionUsageSummary `json:"summary"` + Items []InstanceSessionUsageOverviewItem `json:"items"` + Total int `json:"total"` + Page int `json:"page"` + Limit int `json:"limit"` +} + // AIObservabilityService provides read APIs for audit and cost reporting. type AIObservabilityService interface { ListAuditItems(query AuditQuery) (*AuditListResult, error) GetTraceDetail(traceID string) (*AuditTraceDetail, error) GetCostOverview(query CostQuery) (*CostOverview, error) + GetInstanceSessionUsage(instanceID int, query InstanceSessionUsageQuery) (*InstanceSessionUsageResult, error) + GetInstanceSessionUsageDetail(instanceID int, sessionID string, filter repository.SessionUsageFilter) (*InstanceSessionUsageDetail, error) + GetInstanceLLMGovernanceStatus(instanceID int, runtimeSystemInfo map[string]interface{}) (*InstanceLLMGovernanceStatus, error) + GetLLMGovernanceOverview() (*LLMGovernanceOverview, error) + GetAdminSessionUsageOverview(query InstanceSessionUsageOverviewQuery) (*InstanceSessionUsageOverview, error) } type aiObservabilityService struct { @@ -173,9 +316,11 @@ type aiObservabilityService struct { costRepo repository.CostRecordRepository riskHitRepo repository.RiskHitRepository chatMessageRepo repository.ChatMessageRepository + chatSessionRepo repository.ChatSessionRepository llmModelRepo repository.LLMModelRepository - instanceRepo repository.InstanceRepository - userRepo repository.UserRepository + instanceRepo repository.InstanceRepository + userRepo repository.UserRepository + runtimeStatusRepo repository.InstanceRuntimeStatusRepository } // NewAIObservabilityService creates a new observability reporting service. @@ -185,19 +330,23 @@ func NewAIObservabilityService( costRepo repository.CostRecordRepository, riskHitRepo repository.RiskHitRepository, chatMessageRepo repository.ChatMessageRepository, + chatSessionRepo repository.ChatSessionRepository, llmModelRepo repository.LLMModelRepository, instanceRepo repository.InstanceRepository, userRepo repository.UserRepository, + runtimeStatusRepo repository.InstanceRuntimeStatusRepository, ) AIObservabilityService { return &aiObservabilityService{ - invocationRepo: invocationRepo, - auditRepo: auditRepo, - costRepo: costRepo, - riskHitRepo: riskHitRepo, - chatMessageRepo: chatMessageRepo, - llmModelRepo: llmModelRepo, - instanceRepo: instanceRepo, - userRepo: userRepo, + invocationRepo: invocationRepo, + auditRepo: auditRepo, + costRepo: costRepo, + riskHitRepo: riskHitRepo, + chatMessageRepo: chatMessageRepo, + chatSessionRepo: chatSessionRepo, + llmModelRepo: llmModelRepo, + instanceRepo: instanceRepo, + userRepo: userRepo, + runtimeStatusRepo: runtimeStatusRepo, } } @@ -1847,3 +1996,474 @@ func valueOrCostTotalTokens(cost *models.CostRecord) int { func pointerToInt(value int) *int { return &value } + +func (s *aiObservabilityService) GetInstanceSessionUsage(instanceID int, query InstanceSessionUsageQuery) (*InstanceSessionUsageResult, error) { + page, limit := normalizePageLimit(query.Page, query.Limit, 20, 100) + filter := repository.SessionUsageFilter{Since: query.Since, Until: query.Until} + allItems, err := s.mergeInstanceSessionUsageItems(instanceID, filter) + if err != nil { + return nil, err + } + items := filterSessionUsageItems(allItems, query.Search) + + summary, compliance := summarizeSessionUsageItems(allItems, filter, instanceID, s.invocationRepo, s.auditRepo) + + total := len(items) + start := (page - 1) * limit + if start > total { + start = total + } + end := start + limit + if end > total { + end = total + } + paged := make([]InstanceSessionUsageItem, 0, end-start) + if start < end { + paged = append(paged, items[start:end]...) + } + + return &InstanceSessionUsageResult{ + Summary: summary, + Compliance: compliance, + Items: paged, + Total: total, + Page: page, + Limit: limit, + }, nil +} + +func (s *aiObservabilityService) GetInstanceSessionUsageDetail(instanceID int, sessionID string, filter repository.SessionUsageFilter) (*InstanceSessionUsageDetail, error) { + sessionID = strings.TrimSpace(sessionID) + if sessionID == "" { + return nil, fmt.Errorf("session id is required") + } + + items, err := s.mergeInstanceSessionUsageItems(instanceID, filter) + if err != nil { + return nil, err + } + var matched *InstanceSessionUsageItem + for index := range items { + if items[index].SessionID == sessionID { + matched = &items[index] + break + } + } + if matched == nil { + return nil, fmt.Errorf("session usage not found") + } + + invocations, err := s.invocationRepo.ListRecentByInstanceSession(instanceID, sessionID, 20, filter) + if err != nil { + return nil, fmt.Errorf("failed to list recent invocations for session: %w", err) + } + + modelTotals := map[string]*CostBreakdownItem{} + recentTraces := make([]InstanceSessionTrace, 0, len(invocations)) + for _, invocation := range invocations { + modelRow := modelTotals[invocation.RequestedModel] + if modelRow == nil { + modelRow = &CostBreakdownItem{Label: invocation.RequestedModel} + modelTotals[invocation.RequestedModel] = modelRow + } + modelRow.PromptTokens += invocation.PromptTokens + modelRow.CompletionTokens += invocation.CompletionTokens + modelRow.TotalTokens += invocation.TotalTokens + if s.costRepo != nil { + if costs, costErr := s.costRepo.ListByTraceID(invocation.TraceID); costErr == nil { + for _, cost := range costs { + modelRow.EstimatedCost += cost.EstimatedCost + } + } + } + + recentTraces = append(recentTraces, InstanceSessionTrace{ + TraceID: invocation.TraceID, + RequestedModel: invocation.RequestedModel, + Status: invocation.Status, + PromptTokens: invocation.PromptTokens, + CompletionTokens: invocation.CompletionTokens, + TotalTokens: invocation.TotalTokens, + CreatedAt: invocation.CreatedAt, + }) + } + + return &InstanceSessionUsageDetail{ + SessionID: matched.SessionID, + SessionKey: matched.SessionKey, + Title: matched.Title, + PromptTokens: matched.PromptTokens, + CompletionTokens: matched.CompletionTokens, + TotalTokens: matched.TotalTokens, + EstimatedCost: matched.EstimatedCost, + Currency: matched.Currency, + InvocationCount: matched.InvocationCount, + FirstSeenAt: matched.FirstSeenAt, + LastSeenAt: matched.LastSeenAt, + ModelBreakdown: s.completeModelBreakdowns(modelTotals), + RecentTraces: recentTraces, + }, nil +} + +func (s *aiObservabilityService) GetInstanceLLMGovernanceStatus(instanceID int, runtimeSystemInfo map[string]interface{}) (*InstanceLLMGovernanceStatus, error) { + items, err := s.mergeInstanceSessionUsageItems(instanceID, repository.SessionUsageFilter{}) + if err != nil { + return nil, err + } + + fallbackCount := 0 + for _, item := range items { + if utils.IsTraceFallbackSessionID(item.SessionID) { + fallbackCount++ + } + } + + fallbackRate := 0.0 + if len(items) > 0 { + fallbackRate = float64(fallbackCount) / float64(len(items)) + } + + configStatus := classifyLLMConfigStatusFromSystemInfo(runtimeSystemInfo) + since := time.Now().UTC().Add(-24 * time.Hour) + egressBlockCount := 0 + if s.auditRepo != nil { + if count, err := s.auditRepo.CountRecentByInstanceAndEventType(instanceID, "egress.llm.blocked", since); err == nil { + egressBlockCount = count + } + } + + isCompliant := fallbackRate == 0 && configStatus != "external" + if configStatus == "gateway" { + isCompliant = fallbackRate == 0 + } + + status := &InstanceLLMGovernanceStatus{ + ConfigStatus: configStatus, + SessionFallbackRate: fallbackRate, + RecentEgressBlockCount: egressBlockCount, + IsCompliant: isCompliant, + } + return status, nil +} + +func (s *aiObservabilityService) GetLLMGovernanceOverview() (*LLMGovernanceOverview, error) { + if s.instanceRepo == nil { + return &LLMGovernanceOverview{Items: []LLMGovernanceOverviewItem{}}, nil + } + + instances, err := s.instanceRepo.GetAllRunning() + if err != nil { + return nil, fmt.Errorf("failed to list running instances: %w", err) + } + + overview := &LLMGovernanceOverview{ + Items: make([]LLMGovernanceOverviewItem, 0), + } + for _, instance := range instances { + if !supportsManagedRuntimeIntegration(instance.Type) { + continue + } + systemInfo := decodeRuntimeSystemInfo(s.runtimeStatusRepo, instance.ID) + status, err := s.GetInstanceLLMGovernanceStatus(instance.ID, systemInfo) + if err != nil { + return nil, err + } + if status == nil { + continue + } + + item := LLMGovernanceOverviewItem{ + InstanceID: instance.ID, + InstanceName: instance.Name, + InstanceType: instance.Type, + UserID: instance.UserID, + ConfigStatus: status.ConfigStatus, + SessionFallbackRate: status.SessionFallbackRate, + RecentEgressBlockCount: status.RecentEgressBlockCount, + IsCompliant: status.IsCompliant, + } + overview.TotalManagedInstances++ + overview.Items = append(overview.Items, item) + if !status.IsCompliant { + overview.NonCompliantCount++ + } + if status.ConfigStatus == "external" { + overview.ExternalConfigCount++ + } + if status.SessionFallbackRate > 0 { + overview.HighFallbackCount++ + } + } + + return overview, nil +} + +func (s *aiObservabilityService) GetAdminSessionUsageOverview(query InstanceSessionUsageOverviewQuery) (*InstanceSessionUsageOverview, error) { + if s.instanceRepo == nil { + return &InstanceSessionUsageOverview{Items: []InstanceSessionUsageOverviewItem{}}, nil + } + + page, limit := normalizePageLimit(query.Page, query.Limit, 20, 100) + filter := repository.SessionUsageFilter{Since: query.Since, Until: query.Until} + search := strings.ToLower(strings.TrimSpace(query.Search)) + + instances, err := s.instanceRepo.GetAllRunning() + if err != nil { + return nil, fmt.Errorf("failed to list running instances: %w", err) + } + + allItems := make([]InstanceSessionUsageOverviewItem, 0) + globalSummary := InstanceSessionUsageSummary{Currency: "USD"} + globalCurrencyCounts := map[string]int{} + for _, instance := range instances { + if !supportsManagedRuntimeIntegration(instance.Type) { + continue + } + if search != "" { + nameMatch := strings.Contains(strings.ToLower(instance.Name), search) + idMatch := strings.Contains(strconv.Itoa(instance.ID), search) + typeMatch := strings.Contains(strings.ToLower(instance.Type), search) + if !nameMatch && !idMatch && !typeMatch { + continue + } + } + sessionItems, mergeErr := s.mergeInstanceSessionUsageItems(instance.ID, filter) + if mergeErr != nil { + return nil, mergeErr + } + summary, compliance := summarizeSessionUsageItems(sessionItems, filter, instance.ID, s.invocationRepo, s.auditRepo) + item := InstanceSessionUsageOverviewItem{ + InstanceID: instance.ID, + InstanceName: instance.Name, + InstanceType: instance.Type, + UserID: instance.UserID, + Summary: summary, + Compliance: compliance, + } + allItems = append(allItems, item) + + globalSummary.TotalPromptTokens += summary.TotalPromptTokens + globalSummary.TotalCompletionTokens += summary.TotalCompletionTokens + globalSummary.TotalTokens += summary.TotalTokens + globalSummary.TotalEstimatedCost += summary.TotalEstimatedCost + globalSummary.SessionCount += summary.SessionCount + if strings.TrimSpace(summary.Currency) != "" { + globalCurrencyCounts[summary.Currency]++ + } + } + if len(globalCurrencyCounts) > 0 { + globalSummary.Currency = pickDominantCurrency(globalCurrencyCounts) + } + + sort.Slice(allItems, func(i, j int) bool { + if allItems[i].Summary.TotalTokens == allItems[j].Summary.TotalTokens { + return allItems[i].InstanceID > allItems[j].InstanceID + } + return allItems[i].Summary.TotalTokens > allItems[j].Summary.TotalTokens + }) + + total := len(allItems) + start := (page - 1) * limit + if start > total { + start = total + } + end := start + limit + if end > total { + end = total + } + paged := make([]InstanceSessionUsageOverviewItem, 0, end-start) + if start < end { + paged = append(paged, allItems[start:end]...) + } + + return &InstanceSessionUsageOverview{ + Summary: globalSummary, + Items: paged, + Total: total, + Page: page, + Limit: limit, + }, nil +} + +func summarizeSessionUsageItems( + allItems []InstanceSessionUsageItem, + filter repository.SessionUsageFilter, + instanceID int, + invocationRepo repository.ModelInvocationRepository, + auditRepo repository.AuditEventRepository, +) (InstanceSessionUsageSummary, InstanceSessionUsageCompliance) { + summary := InstanceSessionUsageSummary{Currency: "USD"} + compliance := InstanceSessionUsageCompliance{} + currencyCounts := map[string]int{} + for _, item := range allItems { + summary.TotalPromptTokens += item.PromptTokens + summary.TotalCompletionTokens += item.CompletionTokens + summary.TotalTokens += item.TotalTokens + summary.TotalEstimatedCost += item.EstimatedCost + if strings.TrimSpace(item.Currency) != "" { + currencyCounts[item.Currency]++ + } + if utils.IsTraceFallbackSessionID(item.SessionID) { + compliance.FallbackSessionCount++ + } + } + summary.SessionCount = len(allItems) + if invocationRepo != nil { + if count, countErr := invocationRepo.CountDistinctSessionsByInstance(instanceID, filter); countErr == nil { + summary.SessionCount = count + } + } + compliance.HasFallbackSessions = compliance.FallbackSessionCount > 0 + if auditRepo != nil { + since := time.Now().UTC().Add(-24 * time.Hour) + if filter.Since != nil && filter.Since.After(since) { + since = *filter.Since + } + if count, err := auditRepo.CountRecentByInstanceAndEventType(instanceID, "gateway.session.fallback", since); err == nil { + compliance.RecentFallbackAuditCount = count + } + } + if len(currencyCounts) > 0 { + summary.Currency = pickDominantCurrency(currencyCounts) + } + return summary, compliance +} + +func pickDominantCurrency(currencyCounts map[string]int) string { + bestCurrency := "USD" + bestCount := 0 + for currency, count := range currencyCounts { + if count > bestCount { + bestCurrency = currency + bestCount = count + } + } + return bestCurrency +} + +func decodeRuntimeSystemInfo(runtimeRepo repository.InstanceRuntimeStatusRepository, instanceID int) map[string]interface{} { + if runtimeRepo == nil { + return nil + } + status, err := runtimeRepo.GetByInstanceID(instanceID) + if err != nil || status == nil || status.SystemInfoJSON == nil { + return nil + } + raw := strings.TrimSpace(*status.SystemInfoJSON) + if raw == "" { + return nil + } + systemInfo := map[string]interface{}{} + if err := json.Unmarshal([]byte(raw), &systemInfo); err != nil { + return nil + } + return systemInfo +} + +func (s *aiObservabilityService) mergeInstanceSessionUsageItems(instanceID int, filter repository.SessionUsageFilter) ([]InstanceSessionUsageItem, error) { + tokenAggs, err := s.invocationRepo.AggregateByInstanceSession(instanceID, filter) + if err != nil { + return nil, fmt.Errorf("failed to aggregate session tokens: %w", err) + } + costAggs, err := s.costRepo.AggregateCostByInstanceSession(instanceID, filter) + if err != nil { + return nil, fmt.Errorf("failed to aggregate session costs: %w", err) + } + + sessionMeta := map[string]models.ChatSession{} + if s.chatSessionRepo != nil { + sessions, listErr := s.chatSessionRepo.ListByInstanceID(instanceID) + if listErr != nil { + return nil, fmt.Errorf("failed to list chat sessions: %w", listErr) + } + for _, session := range sessions { + sessionMeta[session.SessionID] = session + } + } + + costBySession := map[string]repository.InstanceSessionCostAggregate{} + for _, cost := range costAggs { + costBySession[cost.SessionID] = cost + } + + items := make([]InstanceSessionUsageItem, 0, len(tokenAggs)) + for _, token := range tokenAggs { + item := InstanceSessionUsageItem{ + SessionID: token.SessionID, + SessionKey: utils.FormatOpenClawSessionKey(token.SessionID), + PromptTokens: token.PromptTokens, + CompletionTokens: token.CompletionTokens, + TotalTokens: token.TotalTokens, + InvocationCount: token.InvocationCount, + FirstSeenAt: token.FirstSeenAt, + LastSeenAt: token.LastSeenAt, + Currency: "USD", + } + if cost, ok := costBySession[token.SessionID]; ok { + item.EstimatedCost = cost.EstimatedCost + if strings.TrimSpace(cost.Currency) != "" { + item.Currency = cost.Currency + } + } + if session, ok := sessionMeta[token.SessionID]; ok && session.Title != nil { + item.Title = session.Title + } + items = append(items, item) + } + + sort.Slice(items, func(i, j int) bool { + return items[i].LastSeenAt.After(items[j].LastSeenAt) + }) + return items, nil +} + +func filterSessionUsageItems(items []InstanceSessionUsageItem, search string) []InstanceSessionUsageItem { + search = strings.ToLower(strings.TrimSpace(search)) + if search == "" { + return items + } + filtered := make([]InstanceSessionUsageItem, 0, len(items)) + for _, item := range items { + haystacks := []string{ + strings.ToLower(item.SessionID), + strings.ToLower(item.SessionKey), + } + if item.Title != nil { + haystacks = append(haystacks, strings.ToLower(*item.Title)) + } + matched := false + for _, candidate := range haystacks { + if strings.Contains(candidate, search) { + matched = true + break + } + } + if matched { + filtered = append(filtered, item) + } + } + return filtered +} + +func classifyLLMConfigStatusFromSystemInfo(systemInfo map[string]interface{}) string { + if len(systemInfo) == 0 { + return "unknown" + } + if raw, ok := systemInfo["llm_config_status"]; ok { + if value, ok := raw.(string); ok && strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + if raw, ok := systemInfo["llm_provider_base_url"]; ok { + if value, ok := raw.(string); ok { + lower := strings.ToLower(strings.TrimSpace(value)) + switch { + case strings.Contains(lower, "gateway/llm"), strings.Contains(lower, "clawmanager"): + return "gateway" + case strings.Contains(lower, "openai.com"), strings.Contains(lower, "anthropic.com"): + return "external" + } + } + } + return "unknown" +} diff --git a/backend/internal/services/ai_observability_session_usage_test.go b/backend/internal/services/ai_observability_session_usage_test.go new file mode 100644 index 0000000..5cd2932 --- /dev/null +++ b/backend/internal/services/ai_observability_session_usage_test.go @@ -0,0 +1,400 @@ +package services + +import ( + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +type stubSessionUsageInvocationRepo struct { + aggregates []repository.InstanceSessionTokenAggregate + invocations []models.ModelInvocation +} + +func (s *stubSessionUsageInvocationRepo) Create(*models.ModelInvocation) error { return nil } +func (s *stubSessionUsageInvocationRepo) GetByID(int) (*models.ModelInvocation, error) { + return nil, nil +} +func (s *stubSessionUsageInvocationRepo) ListByTraceID(string) ([]models.ModelInvocation, error) { + return nil, nil +} +func (s *stubSessionUsageInvocationRepo) ListBySessionID(string, int) ([]models.ModelInvocation, error) { + return nil, nil +} +func (s *stubSessionUsageInvocationRepo) ListByUserID(int, int) ([]models.ModelInvocation, error) { + return nil, nil +} +func (s *stubSessionUsageInvocationRepo) ListRecent(int) ([]models.ModelInvocation, error) { + return nil, nil +} +func (s *stubSessionUsageInvocationRepo) AggregateByInstanceSession(int, repository.SessionUsageFilter) ([]repository.InstanceSessionTokenAggregate, error) { + return s.aggregates, nil +} +func (s *stubSessionUsageInvocationRepo) ListRecentByInstanceSession(int, string, int, repository.SessionUsageFilter) ([]models.ModelInvocation, error) { + return s.invocations, nil +} +func (s *stubSessionUsageInvocationRepo) CountDistinctSessionsByInstance(int, repository.SessionUsageFilter) (int, error) { + return len(s.aggregates), nil +} + +type stubSessionUsageCostRepo struct { + aggregates []repository.InstanceSessionCostAggregate + byTraceID map[string][]models.CostRecord +} + +func (s *stubSessionUsageCostRepo) Create(*models.CostRecord) error { return nil } +func (s *stubSessionUsageCostRepo) ListByTraceID(traceID string) ([]models.CostRecord, error) { + if s.byTraceID == nil { + return nil, nil + } + return s.byTraceID[traceID], nil +} +func (s *stubSessionUsageCostRepo) ListByUserID(int, int) ([]models.CostRecord, error) { + return nil, nil +} +func (s *stubSessionUsageCostRepo) ListRecent(int) ([]models.CostRecord, error) { return nil, nil } +func (s *stubSessionUsageCostRepo) AggregateCostByInstanceSession(int, repository.SessionUsageFilter) ([]repository.InstanceSessionCostAggregate, error) { + return s.aggregates, nil +} + +type stubSessionUsageChatSessionRepo struct { + sessions []models.ChatSession +} + +func (s *stubSessionUsageChatSessionRepo) GetBySessionID(string) (*models.ChatSession, error) { + return nil, nil +} +func (s *stubSessionUsageChatSessionRepo) ListByInstanceID(int) ([]models.ChatSession, error) { + return s.sessions, nil +} +func (s *stubSessionUsageChatSessionRepo) Save(*models.ChatSession) error { return nil } + +type stubSessionUsageAuditRepo struct { + counts map[string]int +} + +func (s *stubSessionUsageAuditRepo) Create(*models.AuditEvent) error { return nil } +func (s *stubSessionUsageAuditRepo) ListByTraceID(string) ([]models.AuditEvent, error) { + return nil, nil +} +func (s *stubSessionUsageAuditRepo) ListRecent(int) ([]models.AuditEvent, error) { + return nil, nil +} +func (s *stubSessionUsageAuditRepo) CountRecentByInstanceAndEventType(instanceID int, eventType string, since time.Time) (int, error) { + if s.counts == nil { + return 0, nil + } + return s.counts[eventType], nil +} + +func TestGetInstanceSessionUsageMergesInvocationCostAndSessionMetadata(t *testing.T) { + now := time.Date(2026, 7, 2, 10, 0, 0, 0, time.UTC) + title := "Weather chat" + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + { + SessionID: "agent:openclaw:main", + PromptTokens: 100, + CompletionTokens: 40, + TotalTokens: 140, + InvocationCount: 2, + FirstSeenAt: now.Add(-time.Hour), + LastSeenAt: now, + }, + }, + }, + costRepo: &stubSessionUsageCostRepo{ + aggregates: []repository.InstanceSessionCostAggregate{ + { + SessionID: "agent:openclaw:main", + EstimatedCost: 0.12, + Currency: "USD", + }, + }, + }, + chatSessionRepo: &stubSessionUsageChatSessionRepo{ + sessions: []models.ChatSession{ + {SessionID: "agent:openclaw:main", Title: &title}, + }, + }, + } + + result, err := service.GetInstanceSessionUsage(9, InstanceSessionUsageQuery{Page: 1, Limit: 10}) + if err != nil { + t.Fatalf("GetInstanceSessionUsage failed: %v", err) + } + if len(result.Items) != 1 { + t.Fatalf("expected 1 item, got %d", len(result.Items)) + } + item := result.Items[0] + if item.SessionKey != "main" || item.TotalTokens != 140 || item.EstimatedCost != 0.12 { + t.Fatalf("unexpected merged item: %+v", item) + } + if item.Title == nil || *item.Title != title { + t.Fatalf("expected title merge, got %+v", item.Title) + } +} + +func TestGetInstanceSessionUsageSearchFiltersBySessionKey(t *testing.T) { + now := time.Now().UTC() + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + {SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now}, + {SessionID: "agent:openclaw:research", TotalTokens: 20, LastSeenAt: now}, + }, + }, + costRepo: &stubSessionUsageCostRepo{}, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + } + + result, err := service.GetInstanceSessionUsage(9, InstanceSessionUsageQuery{Page: 1, Limit: 10, Search: "research"}) + if err != nil { + t.Fatalf("GetInstanceSessionUsage failed: %v", err) + } + if len(result.Items) != 1 || result.Items[0].SessionKey != "research" { + t.Fatalf("unexpected filtered items: %+v", result.Items) + } + if result.Summary.SessionCount != 2 || result.Summary.TotalTokens != 30 { + t.Fatalf("summary should ignore search filter, got %+v", result.Summary) + } +} + +func TestGetInstanceSessionUsageComplianceCountsFallbackSessions(t *testing.T) { + now := time.Now().UTC() + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + {SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now}, + {SessionID: "sess_trc_abc", TotalTokens: 5, LastSeenAt: now}, + }, + }, + costRepo: &stubSessionUsageCostRepo{}, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + } + + result, err := service.GetInstanceSessionUsage(9, InstanceSessionUsageQuery{Page: 1, Limit: 10}) + if err != nil { + t.Fatalf("GetInstanceSessionUsage failed: %v", err) + } + if !result.Compliance.HasFallbackSessions || result.Compliance.FallbackSessionCount != 1 { + t.Fatalf("unexpected compliance: %+v", result.Compliance) + } +} + +func TestGetInstanceSessionUsageComplianceIncludesFallbackAuditCount(t *testing.T) { + now := time.Now().UTC() + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + {SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now}, + }, + }, + costRepo: &stubSessionUsageCostRepo{}, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + auditRepo: &stubSessionUsageAuditRepo{ + counts: map[string]int{"gateway.session.fallback": 3}, + }, + } + + result, err := service.GetInstanceSessionUsage(9, InstanceSessionUsageQuery{Page: 1, Limit: 10}) + if err != nil { + t.Fatalf("GetInstanceSessionUsage failed: %v", err) + } + if result.Compliance.RecentFallbackAuditCount != 3 { + t.Fatalf("expected recent fallback audit count 3, got %+v", result.Compliance) + } +} + +func TestGetInstanceSessionUsageDetailBuildsModelBreakdown(t *testing.T) { + now := time.Now().UTC() + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + { + SessionID: "agent:openclaw:main", + TotalTokens: 30, + LastSeenAt: now, + FirstSeenAt: now, + InvocationCount: 1, + }, + }, + invocations: []models.ModelInvocation{ + { + TraceID: "trc_1", + RequestedModel: "auto", + Status: models.ModelInvocationStatusCompleted, + PromptTokens: 20, + CompletionTokens: 10, + TotalTokens: 30, + CreatedAt: now, + }, + }, + }, + costRepo: &stubSessionUsageCostRepo{ + byTraceID: map[string][]models.CostRecord{ + "trc_1": { + {TraceID: "trc_1", EstimatedCost: 0.05, Currency: "USD"}, + }, + }, + }, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + llmModelRepo: &stubLLMModelRepository{}, + } + + detail, err := service.GetInstanceSessionUsageDetail(9, "agent:openclaw:main", repository.SessionUsageFilter{}) + if err != nil { + t.Fatalf("GetInstanceSessionUsageDetail failed: %v", err) + } + if len(detail.ModelBreakdown) != 1 || detail.ModelBreakdown[0].Label != "auto" { + t.Fatalf("unexpected model breakdown: %+v", detail.ModelBreakdown) + } + if detail.ModelBreakdown[0].EstimatedCost != 0.05 { + t.Fatalf("expected model breakdown cost 0.05, got %+v", detail.ModelBreakdown[0]) + } + if len(detail.RecentTraces) != 1 || detail.RecentTraces[0].TraceID != "trc_1" { + t.Fatalf("unexpected recent traces: %+v", detail.RecentTraces) + } +} + +func TestGetInstanceLLMGovernanceStatusUnknownConfigUsesFallbackOnly(t *testing.T) { + now := time.Now().UTC() + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + {SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now, FirstSeenAt: now}, + }, + }, + costRepo: &stubSessionUsageCostRepo{}, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + auditRepo: &stubSessionUsageAuditRepo{ + counts: map[string]int{"egress.llm.blocked": 2}, + }, + } + + status, err := service.GetInstanceLLMGovernanceStatus(9, map[string]interface{}{}) + if err != nil { + t.Fatalf("GetInstanceLLMGovernanceStatus failed: %v", err) + } + if !status.IsCompliant || status.ConfigStatus != "unknown" || status.RecentEgressBlockCount != 2 { + t.Fatalf("unexpected governance status: %+v", status) + } +} + +func TestGetInstanceLLMGovernanceStatusExternalConfigIsNonCompliant(t *testing.T) { + now := time.Now().UTC() + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + {SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now, FirstSeenAt: now}, + }, + }, + costRepo: &stubSessionUsageCostRepo{}, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + } + + status, err := service.GetInstanceLLMGovernanceStatus(9, map[string]interface{}{ + "llm_config_status": "external", + }) + if err != nil { + t.Fatalf("GetInstanceLLMGovernanceStatus failed: %v", err) + } + if status.IsCompliant { + t.Fatalf("expected external config to be non-compliant") + } +} + +func TestGetAdminSessionUsageOverviewAggregatesManagedInstances(t *testing.T) { + now := time.Now().UTC() + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + {SessionID: "agent:openclaw:main", TotalTokens: 100, LastSeenAt: now, FirstSeenAt: now}, + }, + }, + costRepo: &stubSessionUsageCostRepo{}, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + instanceRepo: &stubGovernanceInstanceRepo{ + instances: []models.Instance{ + {ID: 1, UserID: 9, Name: "oc-1", Type: "openclaw", Status: "running"}, + {ID: 2, UserID: 10, Name: "ubuntu", Type: "ubuntu", Status: "running"}, + }, + }, + } + + overview, err := service.GetAdminSessionUsageOverview(InstanceSessionUsageOverviewQuery{Page: 1, Limit: 10}) + if err != nil { + t.Fatalf("GetAdminSessionUsageOverview failed: %v", err) + } + if overview.Total != 1 || len(overview.Items) != 1 { + t.Fatalf("expected 1 managed instance item, got total=%d items=%d", overview.Total, len(overview.Items)) + } + if overview.Summary.TotalTokens != 100 { + t.Fatalf("unexpected global summary: %+v", overview.Summary) + } +} + +func TestGetAdminSessionUsageOverviewFiltersBySearch(t *testing.T) { + now := time.Now().UTC() + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + {SessionID: "agent:openclaw:main", TotalTokens: 100, LastSeenAt: now, FirstSeenAt: now}, + }, + }, + costRepo: &stubSessionUsageCostRepo{}, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + instanceRepo: &stubGovernanceInstanceRepo{ + instances: []models.Instance{ + {ID: 1, UserID: 9, Name: "alpha-openclaw", Type: "openclaw", Status: "running"}, + {ID: 2, UserID: 9, Name: "beta-openclaw", Type: "openclaw", Status: "running"}, + }, + }, + } + + overview, err := service.GetAdminSessionUsageOverview(InstanceSessionUsageOverviewQuery{ + Page: 1, + Limit: 10, + Search: "alpha", + }) + if err != nil { + t.Fatalf("GetAdminSessionUsageOverview failed: %v", err) + } + if overview.Total != 1 || len(overview.Items) != 1 || overview.Items[0].InstanceName != "alpha-openclaw" { + t.Fatalf("unexpected filtered overview: total=%d items=%+v", overview.Total, overview.Items) + } +} + +func TestGetAdminSessionUsageOverviewAcceptsSinceQuery(t *testing.T) { + now := time.Now().UTC() + since := now.Add(-24 * time.Hour) + service := &aiObservabilityService{ + invocationRepo: &stubSessionUsageInvocationRepo{ + aggregates: []repository.InstanceSessionTokenAggregate{ + {SessionID: "agent:openclaw:main", TotalTokens: 50, LastSeenAt: now, FirstSeenAt: now}, + }, + }, + costRepo: &stubSessionUsageCostRepo{}, + chatSessionRepo: &stubSessionUsageChatSessionRepo{}, + instanceRepo: &stubGovernanceInstanceRepo{ + instances: []models.Instance{ + {ID: 1, UserID: 9, Name: "oc-1", Type: "openclaw", Status: "running"}, + }, + }, + } + + overview, err := service.GetAdminSessionUsageOverview(InstanceSessionUsageOverviewQuery{ + Page: 1, + Limit: 10, + Since: &since, + }) + if err != nil { + t.Fatalf("GetAdminSessionUsageOverview failed: %v", err) + } + if overview.Total != 1 || overview.Summary.TotalTokens != 50 { + t.Fatalf("unexpected since-filtered overview: total=%d summary=%+v", overview.Total, overview.Summary) + } +} diff --git a/backend/internal/services/instance_env.go b/backend/internal/services/instance_env.go index 7d3926a..f7b704d 100644 --- a/backend/internal/services/instance_env.go +++ b/backend/internal/services/instance_env.go @@ -3,6 +3,7 @@ package services import ( "encoding/json" "fmt" + "os" "regexp" "strconv" "strings" @@ -10,6 +11,66 @@ import ( "clawreef/internal/models" ) +var protectedManagedRuntimeEnvKeys = map[string]struct{}{ + "CLAWMANAGER_LLM_BASE_URL": {}, + "CLAWMANAGER_LLM_API_KEY": {}, + "CLAWMANAGER_LLM_MODEL": {}, + "CLAWMANAGER_LLM_PROVIDER": {}, + "CLAWMANAGER_INSTANCE_TOKEN": {}, + "OPENAI_BASE_URL": {}, + "OPENAI_API_BASE": {}, + "OPENAI_API_KEY": {}, + "OPENAI_MODEL": {}, +} + +func isLLMGovernanceStrictEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GOVERNANCE_STRICT"))) { + case "0", "false", "no", "off": + return false + default: + return true + } +} + +func isInstanceNetworkLockEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("CLAWMANAGER_INSTANCE_NETWORK_LOCK"))) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + +func isProtectedManagedRuntimeEnvKey(key string) bool { + _, ok := protectedManagedRuntimeEnvKeys[strings.ToUpper(strings.TrimSpace(key))] + return ok +} + +func validateManagedRuntimeEnvironmentOverrides(instanceType string, overrides map[string]string) error { + if !supportsManagedRuntimeIntegration(instanceType) || !isLLMGovernanceStrictEnabled() { + return nil + } + for key := range overrides { + if isProtectedManagedRuntimeEnvKey(key) { + return fmt.Errorf("environment override %s is managed by the platform", strings.ToUpper(strings.TrimSpace(key))) + } + } + return nil +} + +func applyProtectedManagedRuntimeEnv(target, protected map[string]string) map[string]string { + if len(protected) == 0 { + return target + } + if target == nil { + target = map[string]string{} + } + for key, value := range protected { + target[key] = value + } + return target +} + const ( defaultInstanceSHMSizeGB = 1 maxInstanceSHMSizeGB = 8 @@ -127,6 +188,9 @@ func buildInstancePodEnv(instance *models.Instance, runtimeEnv, gatewayEnv, agen delete(resolved, "SUBFOLDER") } resolved = mergeEnvMaps(resolved, overrides) + if supportsManagedRuntimeIntegration(instance.Type) && isLLMGovernanceStrictEnabled() { + resolved = applyProtectedManagedRuntimeEnv(resolved, mergeEnvMaps(gatewayEnv, agentEnv)) + } return resolved, nil } @@ -145,6 +209,9 @@ func buildInstanceGatewayEnv(instance *models.Instance, gatewayEnv map[string]st resolved = withInstanceProxyEnv(instance.Type, instance.ID, resolved) resolved["CLAWMANAGER_RUNTIME_TYPE"] = normalizeInstanceRuntimeType(instance.RuntimeType) resolved = mergeEnvMaps(resolved, overrides) + if supportsManagedRuntimeIntegration(instance.Type) && isLLMGovernanceStrictEnabled() { + resolved = applyProtectedManagedRuntimeEnv(resolved, gatewayEnv) + } return resolved, nil } diff --git a/backend/internal/services/instance_env_test.go b/backend/internal/services/instance_env_test.go index 9046e6e..7ed1ab3 100644 --- a/backend/internal/services/instance_env_test.go +++ b/backend/internal/services/instance_env_test.go @@ -1,25 +1,44 @@ package services import ( + "strings" "testing" "clawreef/internal/models" ) -func TestNormalizeEnvironmentOverrides(t *testing.T) { - overrides, err := normalizeEnvironmentOverrides(map[string]string{ - " FOO ": "bar", - "BAR_2": "", +func TestValidateManagedRuntimeEnvironmentOverridesRejectsProtectedKeys(t *testing.T) { + err := validateManagedRuntimeEnvironmentOverrides("openclaw", map[string]string{ + "OPENAI_BASE_URL": "https://api.openai.com/v1", + }) + if err == nil { + t.Fatal("expected protected env override to be rejected") + } +} + +func TestApplyProtectedManagedRuntimeEnvRestoresGatewayValues(t *testing.T) { + target := map[string]string{ + "OPENAI_BASE_URL": "https://api.openai.com/v1", + "CUSTOM": "value", + } + protected := map[string]string{ + "OPENAI_BASE_URL": "http://gateway.example/api/v1/gateway/llm", + } + result := applyProtectedManagedRuntimeEnv(target, protected) + if result["OPENAI_BASE_URL"] != protected["OPENAI_BASE_URL"] { + t.Fatalf("expected protected gateway url, got %q", result["OPENAI_BASE_URL"]) + } + if result["CUSTOM"] != "value" { + t.Fatalf("expected custom override to remain") + } +} + +func TestValidateManagedRuntimeEnvironmentOverridesAllowsCustomKeys(t *testing.T) { + err := validateManagedRuntimeEnvironmentOverrides("openclaw", map[string]string{ + "CUSTOM_FLAG": "1", }) if err != nil { - t.Fatalf("normalizeEnvironmentOverrides returned error: %v", err) - } - - if overrides["FOO"] != "bar" { - t.Fatalf("expected trimmed key FOO to be preserved") - } - if value, ok := overrides["BAR_2"]; !ok || value != "" { - t.Fatalf("expected empty override value to be preserved") + t.Fatalf("expected custom override to be allowed, got %v", err) } } @@ -82,73 +101,20 @@ func TestBuildInstancePodEnvAppliesOverridesAfterDefaults(t *testing.T) { } } -func TestBuildInstancePodEnvNormalizesDesktopStreamProfile(t *testing.T) { - raw, err := marshalEnvironmentOverrides(map[string]string{ - "CLAWMANAGER_DESKTOP_STREAM_PROFILE": "standard", - "SELKIES_ENCODER": "x264enc", - "SELKIES_FRAMERATE": "35", - "SELKIES_H264_CRF": "34", +func TestValidateManagedRuntimeEnvironmentOverridesSkipsNonManagedTypes(t *testing.T) { + err := validateManagedRuntimeEnvironmentOverrides("ubuntu", map[string]string{ + "OPENAI_BASE_URL": "https://api.openai.com/v1", }) if err != nil { - t.Fatalf("marshalEnvironmentOverrides returned error: %v", err) - } - - env, err := buildInstancePodEnv(&models.Instance{ - ID: 42, - Type: "openclaw", - RuntimeType: RuntimeBackendDesktop, - EnvironmentOverridesJSON: raw, - }, nil, nil, nil) - if err != nil { - t.Fatalf("buildInstancePodEnv returned error: %v", err) - } - - if got := env["SELKIES_ENCODER"]; got != "x264enc,jpeg" { - t.Fatalf("SELKIES_ENCODER = %q, want x264enc,jpeg", got) - } - if got := env["SELKIES_USE_CSS_SCALING"]; got != "true" { - t.Fatalf("SELKIES_USE_CSS_SCALING = %q, want true", got) + t.Fatalf("expected non-managed type to skip validation, got %v", err) } } -func TestPopSHMSizeGB(t *testing.T) { - tests := []struct { - name string - value string - hasValue bool - runtime string - memoryGB int - want int - }{ - {name: "desktop minimum preset", runtime: "desktop", memoryGB: 4, want: defaultInstanceSHMSizeGB}, - {name: "desktop medium preset", runtime: "desktop", memoryGB: 8, want: 2}, - {name: "desktop large preset", runtime: "desktop", memoryGB: 12, want: 4}, - {name: "desktop small fallback", runtime: "desktop", memoryGB: 2, want: defaultInstanceSHMSizeGB}, - {name: "shell default unchanged", runtime: "shell", memoryGB: 8, want: defaultInstanceSHMSizeGB}, - {name: "disable", value: "0", hasValue: true, runtime: "desktop", memoryGB: 8, want: 0}, - {name: "custom", value: "4", hasValue: true, runtime: "desktop", memoryGB: 4, want: 4}, - {name: "clamp", value: "128", hasValue: true, runtime: "desktop", memoryGB: 8, want: maxInstanceSHMSizeGB}, - {name: "invalid keeps dynamic desktop default", value: "nope", hasValue: true, runtime: "desktop", memoryGB: 8, want: 2}, - {name: "negative keeps dynamic desktop default", value: "-1", hasValue: true, runtime: "desktop", memoryGB: 4, want: defaultInstanceSHMSizeGB}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - extraEnv := map[string]string{"KEEP": "value"} - if tt.hasValue { - extraEnv["SHM_SIZE_GB"] = tt.value - } - - got := popSHMSizeGB(extraEnv, tt.runtime, tt.memoryGB) - if got != tt.want { - t.Fatalf("expected shm size %d, got %d", tt.want, got) - } - if _, ok := extraEnv["SHM_SIZE_GB"]; ok { - t.Fatalf("expected SHM_SIZE_GB to be removed from extra env") - } - if extraEnv["KEEP"] != "value" { - t.Fatalf("expected unrelated env to be preserved") - } - }) +func TestValidateManagedRuntimeEnvironmentOverridesErrorMessage(t *testing.T) { + err := validateManagedRuntimeEnvironmentOverrides("openclaw", map[string]string{ + "openai_api_key": "sk-test", + }) + if err == nil || !strings.Contains(err.Error(), "OPENAI_API_KEY") { + t.Fatalf("expected normalized key in error, got %v", err) } } diff --git a/backend/internal/services/instance_runtime.go b/backend/internal/services/instance_runtime.go index 7cc69e1..6586e57 100644 --- a/backend/internal/services/instance_runtime.go +++ b/backend/internal/services/instance_runtime.go @@ -142,6 +142,7 @@ func withInstanceProxyEnv(instanceType string, instanceID int, env map[string]st merged["https_proxy"] = proxyURL merged["NO_PROXY"] = noProxy merged["no_proxy"] = noProxy + merged["CLAWMANAGER_EGRESS_INSTANCE_ID"] = fmt.Sprintf("%d", instanceID) } if usesWebtopImage(instanceType) { diff --git a/backend/internal/services/instance_runtime_status_service.go b/backend/internal/services/instance_runtime_status_service.go index 25964ac..fad073b 100644 --- a/backend/internal/services/instance_runtime_status_service.go +++ b/backend/internal/services/instance_runtime_status_service.go @@ -19,10 +19,13 @@ type AgentStateReportRequest struct { } type AgentRuntimePayload struct { - OpenClawStatus string `json:"openclaw_status"` - OpenClawPID *int `json:"openclaw_pid,omitempty"` - OpenClawVersion string `json:"openclaw_version"` - CurrentConfigRevisionID *int `json:"current_config_revision_id,omitempty"` + OpenClawStatus string `json:"openclaw_status"` + OpenClawPID *int `json:"openclaw_pid,omitempty"` + OpenClawVersion string `json:"openclaw_version"` + CurrentConfigRevisionID *int `json:"current_config_revision_id,omitempty"` + LLMConfigFingerprint *string `json:"llm_config_fingerprint,omitempty"` + LLMConfigStatus *string `json:"llm_config_status,omitempty"` + LLMProviderBaseURL *string `json:"llm_provider_base_url,omitempty"` } type InstanceRuntimeStatusPayload struct { @@ -90,7 +93,7 @@ func (s *instanceRuntimeStatusService) Report(session *AgentSession, req AgentSt status.CurrentConfigRevisionID = req.Runtime.CurrentConfigRevisionID status.LastReportedAt = reportedAt - systemInfoJSON, err := marshalOptionalJSON(req.SystemInfo) + systemInfoJSON, err := marshalOptionalJSON(mergeLLMConfigIntoSystemInfo(req.SystemInfo, req.Runtime)) if err != nil { return fmt.Errorf("failed to encode system info: %w", err) } @@ -152,6 +155,26 @@ func (s *instanceRuntimeStatusService) GetByInstanceID(instanceID int) (*Instanc return payload, nil } +func mergeLLMConfigIntoSystemInfo(systemInfo map[string]interface{}, runtime AgentRuntimePayload) map[string]interface{} { + merged := map[string]interface{}{} + for key, value := range systemInfo { + merged[key] = value + } + if runtime.LLMConfigFingerprint != nil && strings.TrimSpace(*runtime.LLMConfigFingerprint) != "" { + merged["llm_config_fingerprint"] = strings.TrimSpace(*runtime.LLMConfigFingerprint) + } + if runtime.LLMConfigStatus != nil && strings.TrimSpace(*runtime.LLMConfigStatus) != "" { + merged["llm_config_status"] = strings.TrimSpace(*runtime.LLMConfigStatus) + } + if runtime.LLMProviderBaseURL != nil && strings.TrimSpace(*runtime.LLMProviderBaseURL) != "" { + merged["llm_provider_base_url"] = strings.TrimSpace(*runtime.LLMProviderBaseURL) + } + if len(merged) == 0 { + return nil + } + return merged +} + func (s *instanceRuntimeStatusService) UpsertInfraStatus(instanceID int, infraStatus string) error { status, err := s.getOrCreate(instanceID) if err != nil { diff --git a/backend/internal/services/instance_service.go b/backend/internal/services/instance_service.go index c932e86..1518df5 100644 --- a/backend/internal/services/instance_service.go +++ b/backend/internal/services/instance_service.go @@ -49,6 +49,9 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn if err != nil { return err } + if err := validateManagedRuntimeEnvironmentOverrides(requests[idx].Type, environmentOverrides); err != nil { + return err + } if _, err := marshalEnvironmentOverrides(environmentOverrides); err != nil { return err } @@ -283,6 +286,9 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models if err != nil { return nil, err } + if err := validateManagedRuntimeEnvironmentOverrides(req.Type, environmentOverrides); err != nil { + return nil, err + } if profile, ok := normalizeDesktopStreamProfile(req.DesktopStreamProfile); !ok { return nil, fmt.Errorf("invalid desktop stream profile") } else if profile != "" { @@ -465,26 +471,23 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models var bootstrapSnapshot *models.OpenClawInjectionSnapshot var bootstrapSecretName string - if supportsRuntimeConfigInjection(instance.Type) && s.openClawConfigService != nil && req.OpenClawConfigPlan != nil && hasOpenClawConfigSelections(*req.OpenClawConfigPlan) { - bootstrapSnapshot, err = s.openClawConfigService.CreateSnapshotForInstance(userID, instance, req.OpenClawConfigPlan) - if err != nil { + if snapshot, snapshotErr := s.createRuntimeBootstrapSnapshot(userID, instance, req.OpenClawConfigPlan); snapshotErr != nil { + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to compile runtime bootstrap config: %w", snapshotErr) + } else if snapshot != nil { + bootstrapSnapshot = snapshot + instance.OpenClawConfigSnapshotID = &bootstrapSnapshot.ID + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { s.instanceRepo.Delete(instance.ID) - return nil, fmt.Errorf("failed to compile runtime bootstrap config: %w", err) + return nil, fmt.Errorf("failed to persist runtime snapshot reference: %w", err) } - if bootstrapSnapshot != nil { - instance.OpenClawConfigSnapshotID = &bootstrapSnapshot.ID - instance.UpdatedAt = time.Now() - if err := s.instanceRepo.Update(instance); err != nil { - s.instanceRepo.Delete(instance.ID) - return nil, fmt.Errorf("failed to persist runtime snapshot reference: %w", err) - } - bootstrapSecretName, err = s.openClawConfigService.EnsureSnapshotSecret(ctx, userID, instance, bootstrapSnapshot.ID) - if err != nil { - _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) - s.instanceRepo.Delete(instance.ID) - return nil, fmt.Errorf("failed to provision runtime bootstrap secret: %w", err) - } + bootstrapSecretName, err = s.openClawConfigService.EnsureSnapshotSecret(ctx, userID, instance, bootstrapSnapshot.ID) + if err != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to provision runtime bootstrap secret: %w", err) } } @@ -502,6 +505,14 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models s.instanceRepo.Delete(instance.ID) return nil, fmt.Errorf("failed to create PVC: %w", err) } + if err := EnsureInstanceWorkspacePathForServerScan(ctx, s.instanceRepo, instance); err != nil { + s.pvcService.DeletePVC(ctx, userID, instance.ID) + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + s.instanceRepo.Delete(instance.ID) + return nil, err + } nodeSelector, err := s.pvcService.NodeSelectorForPVC(ctx, userID, instance.ID, storageClass) if err != nil { @@ -513,15 +524,14 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models return nil, fmt.Errorf("failed to resolve PVC node selector: %w", err) } - // Ensure any legacy per-instance network policy is removed before creating pod. - // This keeps new pods unrestricted even if older versions created netpols. - if err := s.networkPolicyService.DeletePolicy(ctx, userID, instance.ID, instance.Name); err != nil { + // Managed runtime network policy: optional egress lock when enabled. + if err := s.syncInstanceNetworkPolicy(ctx, userID, instance); err != nil { s.pvcService.DeletePVC(ctx, userID, instance.ID) if bootstrapSnapshot != nil { _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) } s.instanceRepo.Delete(instance.ID) - return nil, fmt.Errorf("failed to delete network policy: %w", err) + return nil, err } // Create Pod @@ -743,24 +753,20 @@ func (s *instanceService) createV2Instance(ctx context.Context, userID int, req return nil, fmt.Errorf("failed to provision lite agent bootstrap token: %w", err) } - if supportsRuntimeConfigInjection(instance.Type) && s.openClawConfigService != nil && req.OpenClawConfigPlan != nil && hasOpenClawConfigSelections(*req.OpenClawConfigPlan) { - bootstrapSnapshot, err := s.openClawConfigService.CreateSnapshotForInstance(userID, instance, req.OpenClawConfigPlan) - if err != nil { + if snapshot, snapshotErr := s.createRuntimeBootstrapSnapshot(userID, instance, req.OpenClawConfigPlan); snapshotErr != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to compile lite runtime bootstrap config: %w", snapshotErr) + } else if snapshot != nil { + instance.OpenClawConfigSnapshotID = &snapshot.ID + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(snapshot, err) _ = s.instanceRepo.Delete(instance.ID) - return nil, fmt.Errorf("failed to compile lite runtime bootstrap config: %w", err) + return nil, fmt.Errorf("failed to persist lite runtime snapshot reference: %w", err) } - if bootstrapSnapshot != nil { - instance.OpenClawConfigSnapshotID = &bootstrapSnapshot.ID - instance.UpdatedAt = time.Now() - if err := s.instanceRepo.Update(instance); err != nil { - _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) - _ = s.instanceRepo.Delete(instance.ID) - return nil, fmt.Errorf("failed to persist lite runtime snapshot reference: %w", err) - } - if err := s.openClawConfigService.MarkSnapshotActive(bootstrapSnapshot); err != nil { - _ = s.instanceRepo.Delete(instance.ID) - return nil, fmt.Errorf("failed to activate lite runtime bootstrap snapshot: %w", err) - } + if err := s.openClawConfigService.MarkSnapshotActive(snapshot); err != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to activate lite runtime bootstrap snapshot: %w", err) } } @@ -883,6 +889,9 @@ func (s *instanceService) Start(instanceID int) error { if err != nil { return fmt.Errorf("failed to resolve instance environment: %w", err) } + if err := EnsureInstanceWorkspacePathForServerScan(ctx, s.instanceRepo, instance); err != nil { + return err + } bootstrapSecretName := "" if supportsRuntimeConfigInjection(instance.Type) && s.openClawConfigService != nil && instance.OpenClawConfigSnapshotID != nil && *instance.OpenClawConfigSnapshotID > 0 { @@ -1168,6 +1177,7 @@ func (s *instanceService) buildAgentEnv(instance *models.Instance) (map[string]s "CLAWMANAGER_AGENT_INSTANCE_ID": fmt.Sprintf("%d", instance.ID), "CLAWMANAGER_AGENT_PERSISTENT_DIR": managedRuntimePersistentDir(instance), "CLAWMANAGER_AGENT_PROTOCOL_VERSION": AgentProtocolVersionV1, + "CLAWMANAGER_AGENT_RUNTIME_TYPE": strings.ToLower(strings.TrimSpace(instance.Type)), }, nil } @@ -1180,6 +1190,42 @@ func supportsManagedRuntimeIntegration(instanceType string) bool { } } +func (s *instanceService) createRuntimeBootstrapSnapshot(userID int, instance *models.Instance, plan *OpenClawConfigPlan) (*models.OpenClawInjectionSnapshot, error) { + if !supportsRuntimeConfigInjection(instance.Type) || s.openClawConfigService == nil { + return nil, nil + } + if plan != nil && hasOpenClawConfigSelections(*plan) { + return s.openClawConfigService.CreateSnapshotForInstance(userID, instance, plan) + } + if supportsManagedRuntimeIntegration(instance.Type) { + return s.openClawConfigService.CreateDefaultLLMGovernanceSnapshot(userID, instance) + } + return nil, nil +} + +func (s *instanceService) syncInstanceNetworkPolicy(ctx context.Context, userID int, instance *models.Instance) error { + if instance == nil { + return nil + } + if isLiteRuntimeInstance(instance) { + // Lite/gateway-pool instances share runtime pods; per-instance NetworkPolicy does not apply. + if err := s.networkPolicyService.DeletePolicy(ctx, userID, instance.ID, instance.Name); err != nil { + return fmt.Errorf("failed to delete network policy: %w", err) + } + return nil + } + if isInstanceNetworkLockEnabled() && supportsManagedRuntimeIntegration(instance.Type) { + if err := s.networkPolicyService.EnsureDefaultPolicy(ctx, userID, instance.ID, instance.Name); err != nil { + return fmt.Errorf("failed to ensure network policy: %w", err) + } + return nil + } + if err := s.networkPolicyService.DeletePolicy(ctx, userID, instance.ID, instance.Name); err != nil { + return fmt.Errorf("failed to delete network policy: %w", err) + } + return nil +} + func supportsRuntimeConfigInjection(instanceType string) bool { switch strings.ToLower(strings.TrimSpace(instanceType)) { case "openclaw", "hermes": @@ -1794,6 +1840,9 @@ func (s *instanceService) Update(instanceID int, req UpdateInstanceRequest) erro return err } environmentOverrides = applyDesktopStreamProfileEnv(environmentOverrides, profile) + if err := validateManagedRuntimeEnvironmentOverrides(instance.Type, environmentOverrides); err != nil { + return err + } environmentOverridesJSON, err := marshalEnvironmentOverrides(environmentOverrides) if err != nil { return err diff --git a/backend/internal/services/instance_service_v2_test.go b/backend/internal/services/instance_service_v2_test.go index 7ba7d70..396a3aa 100644 --- a/backend/internal/services/instance_service_v2_test.go +++ b/backend/internal/services/instance_service_v2_test.go @@ -698,6 +698,14 @@ func (s *liteOpenClawConfigStub) CreateSnapshotForInstance(userID int, instance return s.snapshot, nil } +func (s *liteOpenClawConfigStub) CreateDefaultLLMGovernanceSnapshot(userID int, instance *models.Instance) (*models.OpenClawInjectionSnapshot, error) { + return s.CreateSnapshotForInstance(userID, instance, nil) +} + +func (s *liteOpenClawConfigStub) EnsurePlatformLLMGatewayResource(userID int) (*models.OpenClawConfigResource, error) { + return nil, nil +} + func (s *liteOpenClawConfigStub) MarkSnapshotActive(snapshot *models.OpenClawInjectionSnapshot) error { s.activated = true if snapshot != nil { diff --git a/backend/internal/services/instance_workspace_path.go b/backend/internal/services/instance_workspace_path.go new file mode 100644 index 0000000..b2dd6b1 --- /dev/null +++ b/backend/internal/services/instance_workspace_path.go @@ -0,0 +1,63 @@ +package services + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" +) + +func hostPathWorkspaceScanEnabled() bool { + client := k8s.GetClient() + return client != nil && client.HostPathFallbackEnabled +} + +func instancePersistentHostPath(userID, instanceID int) (string, bool) { + if !hostPathWorkspaceScanEnabled() || userID <= 0 || instanceID <= 0 { + return "", false + } + hostPathPrefix := "/data/clawreef" + if client := k8s.GetClient(); client != nil && strings.TrimSpace(client.HostPathPrefix) != "" { + hostPathPrefix = strings.TrimSpace(client.HostPathPrefix) + } + return filepath.Join(hostPathPrefix, fmt.Sprintf("user-%d", userID), fmt.Sprintf("instance-%d", instanceID)), true +} + +func proDesktopWorkspaceScanEligible(instance *models.Instance) bool { + if instance == nil || isLiteRuntimeInstance(instance) { + return false + } + if v2Type, ok := v2RuntimeTypeForInstance(instance); ok && strings.TrimSpace(v2Type) != "" { + return false + } + return supportsManagedRuntimeIntegration(instance.Type) +} + +func EnsureInstanceWorkspacePathForServerScan(ctx context.Context, repo repository.InstanceRepository, instance *models.Instance) error { + if repo == nil || instance == nil { + return nil + } + if instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != "" { + return nil + } + if !proDesktopWorkspaceScanEligible(instance) { + return nil + } + hostPath, ok := instancePersistentHostPath(instance.UserID, instance.ID) + if !ok { + return nil + } + if _, err := os.Stat(hostPath); err != nil { + return nil + } + if err := repo.SetWorkspacePath(ctx, instance.ID, hostPath); err != nil { + return fmt.Errorf("failed to persist pro desktop workspace path: %w", err) + } + instance.WorkspacePath = &hostPath + return nil +} diff --git a/backend/internal/services/k8s/instance_deployment_service.go b/backend/internal/services/k8s/instance_deployment_service.go index 34a7ee0..16eadd5 100644 --- a/backend/internal/services/k8s/instance_deployment_service.go +++ b/backend/internal/services/k8s/instance_deployment_service.go @@ -244,6 +244,7 @@ func BuildInstanceDeployment(client *Client, config PodConfig, replicas int32) * "runtime-type": runtimeType, "managed-by": "clawreef", } + appendManagedRuntimeLabels(config.Type, labels) return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ diff --git a/backend/internal/services/k8s/network_policy_service.go b/backend/internal/services/k8s/network_policy_service.go index b0e4021..ecdf54b 100644 --- a/backend/internal/services/k8s/network_policy_service.go +++ b/backend/internal/services/k8s/network_policy_service.go @@ -55,9 +55,10 @@ func (s *NetworkPolicyService) EnsureDefaultPolicy(ctx context.Context, userID, Spec: networkingv1.NetworkPolicySpec{ PodSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": "clawreef", - "instance-id": instanceLabel, - "managed-by": "clawreef", + "app": "clawreef", + "instance-id": instanceLabel, + "managed-by": "clawreef", + "clawmanager.io/managed-runtime": "true", }, }, PolicyTypes: []networkingv1.PolicyType{ diff --git a/backend/internal/services/k8s/pod_service.go b/backend/internal/services/k8s/pod_service.go index 8872e23..ff8bfc8 100644 --- a/backend/internal/services/k8s/pod_service.go +++ b/backend/internal/services/k8s/pod_service.go @@ -156,6 +156,7 @@ func (s *PodService) CreatePod(ctx context.Context, config PodConfig) (*corev1.P "runtime-type": runtimeType, "managed-by": "clawreef", } + appendManagedRuntimeLabels(config.Type, labels) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/backend/internal/services/k8s/runtime_labels.go b/backend/internal/services/k8s/runtime_labels.go new file mode 100644 index 0000000..c4c4bae --- /dev/null +++ b/backend/internal/services/k8s/runtime_labels.go @@ -0,0 +1,13 @@ +package k8s + +import "strings" + +func appendManagedRuntimeLabels(instanceType string, labels map[string]string) { + if labels == nil { + return + } + switch strings.ToLower(strings.TrimSpace(instanceType)) { + case "openclaw", "hermes": + labels["clawmanager.io/managed-runtime"] = "true" + } +} diff --git a/backend/internal/services/openclaw_config_service.go b/backend/internal/services/openclaw_config_service.go index 2f43a1f..715d4ab 100644 --- a/backend/internal/services/openclaw_config_service.go +++ b/backend/internal/services/openclaw_config_service.go @@ -263,6 +263,8 @@ type OpenClawConfigService interface { CompilePreview(userID int, plan OpenClawConfigPlan) (*OpenClawConfigCompilePreview, error) PlanWithoutTeamMemberLeaderOnlyChannels(userID int, plan *OpenClawConfigPlan) (*OpenClawConfigPlan, error) CreateSnapshotForInstance(userID int, instance *models.Instance, plan *OpenClawConfigPlan) (*models.OpenClawInjectionSnapshot, error) + CreateDefaultLLMGovernanceSnapshot(userID int, instance *models.Instance) (*models.OpenClawInjectionSnapshot, error) + EnsurePlatformLLMGatewayResource(userID int) (*models.OpenClawConfigResource, error) MarkSnapshotActive(snapshot *models.OpenClawInjectionSnapshot) error MarkSnapshotFailed(snapshot *models.OpenClawInjectionSnapshot, err error) error EnsureSnapshotSecret(ctx context.Context, userID int, instance *models.Instance, snapshotID int) (string, error) diff --git a/backend/internal/services/openclaw_platform_llm_gateway.go b/backend/internal/services/openclaw_platform_llm_gateway.go new file mode 100644 index 0000000..d5f6a23 --- /dev/null +++ b/backend/internal/services/openclaw_platform_llm_gateway.go @@ -0,0 +1,88 @@ +package services + +import ( + "encoding/json" + "fmt" + "time" + + "clawreef/internal/models" +) + +const ( + PlatformLLMGatewayResourceKey = "platform-llm-gateway" + PlatformLLMGatewayResourceName = "Platform LLM Gateway" +) + +var platformLLMGatewayAgentContent = json.RawMessage(`{ + "schemaVersion": 1, + "kind": "agent", + "format": "agent/platform-llm-gateway@v1", + "dependsOn": [], + "config": { + "models": { + "providers": { + "clawmanager": { + "type": "openai-compatible", + "baseUrl": "${CLAWMANAGER_LLM_BASE_URL}", + "apiKey": "${CLAWMANAGER_LLM_API_KEY}", + "default": true + } + }, + "primary": "auto/auto" + } + } +}`) + +func (s *openClawConfigService) EnsurePlatformLLMGatewayResource(userID int) (*models.OpenClawConfigResource, error) { + if userID <= 0 { + return nil, fmt.Errorf("user id is required") + } + + existing, err := s.repo.GetResourceByUserTypeKey(userID, OpenClawConfigResourceTypeAgent, PlatformLLMGatewayResourceKey) + if err != nil { + return nil, err + } + if existing != nil { + return existing, nil + } + + description := "Built-in agent config that routes OpenClaw LLM calls through the ClawManager AI Gateway." + now := time.Now() + item := &models.OpenClawConfigResource{ + UserID: userID, + ResourceType: OpenClawConfigResourceTypeAgent, + ResourceKey: PlatformLLMGatewayResourceKey, + Name: PlatformLLMGatewayResourceName, + Description: &description, + Enabled: true, + Version: 1, + TagsJSON: encodeStringArray([]string{"builtin", "llm-governance", "platform"}), + ContentJSON: string(platformLLMGatewayAgentContent), + CreatedAt: now, + UpdatedAt: now, + } + if err := s.repo.CreateResource(item); err != nil { + return nil, err + } + return item, nil +} + +func (s *openClawConfigService) CreateDefaultLLMGovernanceSnapshot(userID int, instance *models.Instance) (*models.OpenClawInjectionSnapshot, error) { + if instance == nil || !supportsManagedRuntimeIntegration(instance.Type) { + return nil, nil + } + + resource, err := s.EnsurePlatformLLMGatewayResource(userID) + if err != nil { + return nil, err + } + if resource == nil || resource.ID <= 0 { + return nil, fmt.Errorf("failed to provision platform llm gateway resource") + } + + plan := &OpenClawConfigPlan{ + Mode: OpenClawConfigPlanModeManual, + ResourceIDs: []int{resource.ID}, + } + return s.CreateSnapshotForInstance(userID, instance, plan) +} diff --git a/backend/internal/services/openclaw_platform_llm_gateway_test.go b/backend/internal/services/openclaw_platform_llm_gateway_test.go new file mode 100644 index 0000000..acd59e8 --- /dev/null +++ b/backend/internal/services/openclaw_platform_llm_gateway_test.go @@ -0,0 +1,152 @@ +package services + +import ( + "fmt" + "strings" + "testing" + "time" + + "clawreef/internal/models" +) + +type platformLLMGatewayRepoStub struct { + nextResourceID int + nextSnapshotID int + resources map[string]*models.OpenClawConfigResource +} + +func newPlatformLLMGatewayRepoStub() *platformLLMGatewayRepoStub { + return &platformLLMGatewayRepoStub{ + nextResourceID: 1, + nextSnapshotID: 1, + resources: map[string]*models.OpenClawConfigResource{}, + } +} + +func (s *platformLLMGatewayRepoStub) resourceKey(userID int, resourceType, resourceKey string) string { + return fmt.Sprintf("%d:%s:%s", userID, resourceType, resourceKey) +} + +func (s *platformLLMGatewayRepoStub) ListResources(userID int, resourceType string) ([]models.OpenClawConfigResource, error) { + return nil, nil +} +func (s *platformLLMGatewayRepoStub) GetResourceByID(id int) (*models.OpenClawConfigResource, error) { + for _, resource := range s.resources { + if resource.ID == id { + copy := *resource + return ©, nil + } + } + return nil, nil +} +func (s *platformLLMGatewayRepoStub) GetResourceByUserTypeKey(userID int, resourceType, resourceKey string) (*models.OpenClawConfigResource, error) { + resource := s.resources[s.resourceKey(userID, resourceType, resourceKey)] + if resource == nil { + return nil, nil + } + copy := *resource + return ©, nil +} +func (s *platformLLMGatewayRepoStub) CreateResource(resource *models.OpenClawConfigResource) error { + if resource == nil { + return nil + } + resource.ID = s.nextResourceID + s.nextResourceID++ + s.resources[s.resourceKey(resource.UserID, resource.ResourceType, resource.ResourceKey)] = resource + return nil +} +func (s *platformLLMGatewayRepoStub) UpdateResource(resource *models.OpenClawConfigResource) error { + return nil +} +func (s *platformLLMGatewayRepoStub) DeleteResource(id int) error { return nil } +func (s *platformLLMGatewayRepoStub) ListBundles(userID int) ([]models.OpenClawConfigBundle, error) { + return nil, nil +} +func (s *platformLLMGatewayRepoStub) GetBundleByID(id int) (*models.OpenClawConfigBundle, error) { + return nil, nil +} +func (s *platformLLMGatewayRepoStub) CreateBundle(bundle *models.OpenClawConfigBundle) error { return nil } +func (s *platformLLMGatewayRepoStub) UpdateBundle(bundle *models.OpenClawConfigBundle) error { + return nil +} +func (s *platformLLMGatewayRepoStub) DeleteBundle(id int) error { return nil } +func (s *platformLLMGatewayRepoStub) ListBundleItems(bundleID int) ([]models.OpenClawConfigBundleItem, error) { + return nil, nil +} +func (s *platformLLMGatewayRepoStub) ReplaceBundleItems(bundleID int, items []models.OpenClawConfigBundleItem) error { + return nil +} +func (s *platformLLMGatewayRepoStub) ListBundleSkills(bundleID int) ([]models.OpenClawConfigBundleSkill, error) { + return nil, nil +} +func (s *platformLLMGatewayRepoStub) ReplaceBundleSkills(bundleID int, items []models.OpenClawConfigBundleSkill) error { + return nil +} +func (s *platformLLMGatewayRepoStub) CreateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error { + if snapshot == nil { + return nil + } + snapshot.ID = s.nextSnapshotID + s.nextSnapshotID++ + return nil +} +func (s *platformLLMGatewayRepoStub) UpdateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error { + return nil +} +func (s *platformLLMGatewayRepoStub) GetSnapshotByID(id int) (*models.OpenClawInjectionSnapshot, error) { + return nil, nil +} +func (s *platformLLMGatewayRepoStub) ListSnapshotsByUser(userID int, limit int) ([]models.OpenClawInjectionSnapshot, error) { + return nil, nil +} +func (s *platformLLMGatewayRepoStub) ListActiveSnapshots(userID int) ([]models.OpenClawInjectionSnapshot, error) { + return nil, nil +} +func (s *platformLLMGatewayRepoStub) UpdateSnapshotIfUnchanged(snapshot *models.OpenClawInjectionSnapshot, expectedUpdatedAt time.Time) (bool, error) { + return true, nil +} + +func TestEnsurePlatformLLMGatewayResourceCreatesBuiltinAgentResource(t *testing.T) { + repo := newPlatformLLMGatewayRepoStub() + service := &openClawConfigService{repo: repo} + + resource, err := service.EnsurePlatformLLMGatewayResource(9) + if err != nil { + t.Fatalf("EnsurePlatformLLMGatewayResource returned error: %v", err) + } + if resource == nil || resource.ID <= 0 { + t.Fatalf("expected created resource, got %+v", resource) + } + if resource.ResourceType != OpenClawConfigResourceTypeAgent || resource.ResourceKey != PlatformLLMGatewayResourceKey { + t.Fatalf("unexpected resource identity: %+v", resource) + } + + again, err := service.EnsurePlatformLLMGatewayResource(9) + if err != nil { + t.Fatalf("second EnsurePlatformLLMGatewayResource returned error: %v", err) + } + if again == nil || again.ID != resource.ID { + t.Fatalf("expected same resource id, got %+v want %d", again, resource.ID) + } +} + +func TestCreateDefaultLLMGovernanceSnapshotCompilesPlatformGatewayAgent(t *testing.T) { + repo := newPlatformLLMGatewayRepoStub() + service := &openClawConfigService{repo: repo} + instance := &models.Instance{ID: 42, UserID: 9, Type: "openclaw", Name: "oc-42"} + + snapshot, err := service.CreateDefaultLLMGovernanceSnapshot(9, instance) + if err != nil { + t.Fatalf("CreateDefaultLLMGovernanceSnapshot returned error: %v", err) + } + if snapshot == nil { + t.Fatal("expected snapshot") + } + if snapshot.Mode != OpenClawConfigPlanModeManual { + t.Fatalf("snapshot mode = %q, want manual", snapshot.Mode) + } + if !strings.Contains(snapshot.ResolvedResourcesJSON, PlatformLLMGatewayResourceKey) { + t.Fatalf("expected platform gateway resource in snapshot, got %s", snapshot.ResolvedResourcesJSON) + } +} diff --git a/backend/internal/services/runtime_agent_client.go b/backend/internal/services/runtime_agent_client.go index fcf5b0d..d6d185f 100644 --- a/backend/internal/services/runtime_agent_client.go +++ b/backend/internal/services/runtime_agent_client.go @@ -17,6 +17,7 @@ type RuntimeAgentClient interface { CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error Drain(ctx context.Context, endpoint string) error + ResyncInstanceSkills(ctx context.Context, endpoint string, instanceID int, mode string) error } type RuntimeAgentPortRange struct { @@ -91,6 +92,19 @@ func (c *runtimeAgentHTTPClient) Drain(ctx context.Context, endpoint string) err return c.do(ctx, http.MethodPost, endpoint, "/v1/drain", map[string]bool{"draining": true}, nil) } +func (c *runtimeAgentHTTPClient) ResyncInstanceSkills(ctx context.Context, endpoint string, instanceID int, mode string) error { + mode = strings.TrimSpace(mode) + if mode == "" { + mode = "full" + } + body := map[string]any{ + "instance_id": instanceID, + "mode": mode, + "trigger": "manual", + } + return c.do(ctx, http.MethodPost, endpoint, "/v1/skills/resync", body, nil) +} + func (c *runtimeAgentHTTPClient) do(ctx context.Context, method, endpoint, path string, body any, out any) error { endpoint = strings.TrimRight(endpoint, "/") var reader io.Reader diff --git a/backend/internal/services/runtime_agent_client_test.go b/backend/internal/services/runtime_agent_client_test.go index aae5374..04f56b1 100644 --- a/backend/internal/services/runtime_agent_client_test.go +++ b/backend/internal/services/runtime_agent_client_test.go @@ -192,6 +192,30 @@ func TestRuntimeAgentClientDrainSendsJSONBody(t *testing.T) { } } +func TestRuntimeAgentClientResyncInstanceSkills(t *testing.T) { + var gotMethod, gotPath, gotToken string + var body map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotPath = r.URL.Path + gotToken = r.Header.Get("X-ClawManager-Control-Token") + _ = json.NewDecoder(r.Body).Decode(&body) + w.WriteHeader(http.StatusAccepted) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + if err := client.ResyncInstanceSkills(context.Background(), server.URL, 12, "full"); err != nil { + t.Fatalf("ResyncInstanceSkills returned error: %v", err) + } + if gotMethod != http.MethodPost || gotPath != "/v1/skills/resync" || gotToken != "secret" { + t.Fatalf("unexpected request: %s %s token=%q", gotMethod, gotPath, gotToken) + } + if body["instance_id"] != float64(12) || body["mode"] != "full" { + t.Fatalf("unexpected body: %#v", body) + } +} + func TestRuntimeAgentClientConflict(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "no free port", http.StatusConflict) diff --git a/backend/internal/services/runtime_scheduler_test.go b/backend/internal/services/runtime_scheduler_test.go index 221d987..fd6a6c8 100644 --- a/backend/internal/services/runtime_scheduler_test.go +++ b/backend/internal/services/runtime_scheduler_test.go @@ -2367,6 +2367,9 @@ func (c *fakeRuntimeAgentClient) Drain(ctx context.Context, endpoint string) err c.drainEndpoints = append(c.drainEndpoints, endpoint) return nil } +func (c *fakeRuntimeAgentClient) ResyncInstanceSkills(ctx context.Context, endpoint string, instanceID int, mode string) error { + return nil +} type fakeRuntimeEventService struct { published []fakeRuntimeEvent diff --git a/backend/internal/services/skill_content_md5_test.go b/backend/internal/services/skill_content_md5_test.go new file mode 100644 index 0000000..01bdd57 --- /dev/null +++ b/backend/internal/services/skill_content_md5_test.go @@ -0,0 +1,34 @@ +package services + +import ( + "errors" + "testing" + + "clawreef/internal/utils" +) + +func TestHashDirectoryGoldenWeatherFixture(t *testing.T) { + files := map[string][]byte{ + "src/main.py": []byte("print('weather')\n"), + } + got := hashDirectory(files) + want := referenceSkillContentMD5(files) + if got != want { + t.Fatalf("hashDirectory() = %s, want %s", got, want) + } +} + +func TestHubErrorMD5MismatchCode(t *testing.T) { + err := utils.NewHubError( + "skill_package_md5_mismatch", + "skill package md5 mismatch: expected abc got def", + map[string]string{"expected": "abc", "computed": "def"}, + ) + var hubErr *utils.HubError + if !errors.As(err, &hubErr) { + t.Fatal("expected HubError") + } + if hubErr.Code != "skill_package_md5_mismatch" { + t.Fatalf("expected skill_package_md5_mismatch, got %q", hubErr.Code) + } +} diff --git a/backend/internal/services/skill_hub_repo_stub_test.go b/backend/internal/services/skill_hub_repo_stub_test.go new file mode 100644 index 0000000..8f8dfbc --- /dev/null +++ b/backend/internal/services/skill_hub_repo_stub_test.go @@ -0,0 +1,302 @@ +package services + +import ( + "time" + + "clawreef/internal/models" +) + +type skillRepoStub struct { + skills map[int]*models.Skill + blobs map[int]*models.SkillBlob + versions map[int]*models.SkillVersion + tags map[int]*models.SkillHubTag + tagAssignments map[int][]int + instanceSkillsBySkillID map[int][]models.InstanceSkill + instanceSkills []models.InstanceSkill + hardDeleteCalled bool +} + +func (s *skillRepoStub) ListSkillsByUser(userID int) ([]models.Skill, error) { + items := make([]models.Skill, 0) + for _, skill := range s.skills { + if skill.UserID == userID { + items = append(items, *skill) + } + } + return items, nil +} + +func (s *skillRepoStub) ListAllSkills() ([]models.Skill, error) { + items := make([]models.Skill, 0, len(s.skills)) + for _, skill := range s.skills { + items = append(items, *skill) + } + return items, nil +} + +func (s *skillRepoStub) GetSkillByID(id int) (*models.Skill, error) { + if skill, ok := s.skills[id]; ok { + copy := *skill + return ©, nil + } + return nil, nil +} + +func (s *skillRepoStub) GetSkillByUserKey(userID int, skillKey string) (*models.Skill, error) { + for _, skill := range s.skills { + if skill.UserID == userID && skill.SkillKey == skillKey && skill.Status == skillStatusActive { + copy := *skill + return ©, nil + } + } + return nil, nil +} + +func (s *skillRepoStub) CreateSkill(*models.Skill) error { return nil } +func (s *skillRepoStub) UpdateSkill(skill *models.Skill) error { + if s.skills == nil { + s.skills = map[int]*models.Skill{} + } + copy := *skill + s.skills[skill.ID] = © + return nil +} + +func (s *skillRepoStub) DeleteSkill(int) error { + s.hardDeleteCalled = true + return nil +} + +func (s *skillRepoStub) GetBlobByContentHash(string) (*models.SkillBlob, error) { return nil, nil } + +func (s *skillRepoStub) GetBlobByID(id int) (*models.SkillBlob, error) { + if blob, ok := s.blobs[id]; ok { + copy := *blob + return ©, nil + } + return nil, nil +} + +func (s *skillRepoStub) CreateBlob(blob *models.SkillBlob) error { + if s.blobs == nil { + s.blobs = map[int]*models.SkillBlob{} + } + if blob.ID == 0 { + blob.ID = len(s.blobs) + 1 + } + copy := *blob + s.blobs[blob.ID] = © + return nil +} +func (s *skillRepoStub) UpdateBlob(blob *models.SkillBlob) error { + if s.blobs == nil { + s.blobs = map[int]*models.SkillBlob{} + } + copy := *blob + s.blobs[blob.ID] = © + return nil +} +func (s *skillRepoStub) ListVersionsBySkillID(int) ([]models.SkillVersion, error) { + return nil, nil +} + +func (s *skillRepoStub) GetVersionByID(id int) (*models.SkillVersion, error) { + if version, ok := s.versions[id]; ok { + copy := *version + return ©, nil + } + return nil, nil +} + +func (s *skillRepoStub) GetVersionBySkillAndBlob(int, int) (*models.SkillVersion, error) { + return nil, nil +} + +func (s *skillRepoStub) GetLatestVersionBySkillID(int) (*models.SkillVersion, error) { + return nil, nil +} + +func (s *skillRepoStub) CreateVersion(*models.SkillVersion) error { return nil } + +func (s *skillRepoStub) UpdateVersion(version *models.SkillVersion) error { + if s.versions == nil { + s.versions = map[int]*models.SkillVersion{} + } + copy := *version + s.versions[version.ID] = © + return nil +} + +func (s *skillRepoStub) ListInstanceSkills(int) ([]models.InstanceSkill, error) { + return nil, nil +} + +func (s *skillRepoStub) ListActiveInstanceSkillsBySkillID(skillID int) ([]models.InstanceSkill, error) { + if s.instanceSkillsBySkillID != nil { + if items, ok := s.instanceSkillsBySkillID[skillID]; ok { + return filterActiveInstanceSkills(items), nil + } + } + items := make([]models.InstanceSkill, 0) + for _, item := range s.instanceSkills { + if item.SkillID == skillID && item.Status != "removed" && item.Status != "missing" { + items = append(items, item) + } + } + return items, nil +} + +func filterActiveInstanceSkills(items []models.InstanceSkill) []models.InstanceSkill { + active := make([]models.InstanceSkill, 0, len(items)) + for _, item := range items { + if item.Status != "removed" && item.Status != "missing" { + active = append(active, item) + } + } + return active +} + +func (s *skillRepoStub) GetInstanceSkill(instanceID, skillID int) (*models.InstanceSkill, error) { + for _, item := range s.instanceSkills { + if item.InstanceID == instanceID && item.SkillID == skillID { + copy := item + return ©, nil + } + } + return nil, nil +} +func (s *skillRepoStub) UpsertInstanceSkill(*models.InstanceSkill) error { return nil } +func (s *skillRepoStub) MarkInstanceSkillRemoved(int, int, time.Time) error { return nil } +func (s *skillRepoStub) MarkInstanceSkillRemovedBySkillKey(int, string, time.Time) error { + return nil +} +func (s *skillRepoStub) MarkInstanceSkillsRemovedByWorkspacePath(int, string, time.Time) error { + return nil +} +func (s *skillRepoStub) MarkMissingInstanceSkills(int, []int, time.Time) error { return nil } +func (s *skillRepoStub) CreateScanResult(result *models.SkillScanResult) error { + if result.ID == 0 { + result.ID = 99 + } + return nil +} +func (s *skillRepoStub) GetScanResultByID(int) (*models.SkillScanResult, error) { return nil, nil } +func (s *skillRepoStub) ListScanResultsByBlobID(int) ([]models.SkillScanResult, error) { + return nil, nil +} + +func (s *skillRepoStub) GetLatestScanResultByBlobID(int) (*models.SkillScanResult, error) { + return nil, nil +} + +func (s *skillRepoStub) GetLatestScanResultBySkillID(int) (*models.SkillScanResult, error) { + return nil, nil +} + +func (s *skillRepoStub) ListHubTags(bool) ([]models.SkillHubTag, error) { return nil, nil } + +func (s *skillRepoStub) GetHubTagByID(id int) (*models.SkillHubTag, error) { + if tag, ok := s.tags[id]; ok { + copy := *tag + return ©, nil + } + return nil, nil +} + +func (s *skillRepoStub) ListHubTagsBySkillID(skillID int) ([]models.SkillHubTag, error) { + tagIDs := s.tagAssignments[skillID] + result := make([]models.SkillHubTag, 0, len(tagIDs)) + for _, tagID := range tagIDs { + if tag, ok := s.tags[tagID]; ok { + result = append(result, *tag) + } + } + return result, nil +} + +func (s *skillRepoStub) ReplaceSkillTagAssignments(skillID int, tagIDs []int) error { + if s.tagAssignments == nil { + s.tagAssignments = map[int][]int{} + } + s.tagAssignments[skillID] = append([]int(nil), tagIDs...) + return nil +} + +func (s *skillRepoStub) ListPublicHubSkills() ([]models.Skill, error) { + items := make([]models.Skill, 0) + for _, skill := range s.skills { + if skill.Visibility == skillVisibilityPublic && skill.Status == skillStatusActive { + items = append(items, *skill) + } + } + return items, nil +} + +func (s *skillRepoStub) ListSkillsForHubAdmin() ([]models.Skill, error) { + return s.ListAllSkills() +} + +type hubTagRepoStub struct { + tags map[int]*models.SkillHubTag +} + +func (s *hubTagRepoStub) ListSkillsByUser(int) ([]models.Skill, error) { return nil, nil } +func (s *hubTagRepoStub) ListAllSkills() ([]models.Skill, error) { return nil, nil } +func (s *hubTagRepoStub) GetSkillByID(int) (*models.Skill, error) { return nil, nil } +func (s *hubTagRepoStub) GetSkillByUserKey(int, string) (*models.Skill, error) { return nil, nil } +func (s *hubTagRepoStub) CreateSkill(*models.Skill) error { return nil } +func (s *hubTagRepoStub) UpdateSkill(*models.Skill) error { return nil } +func (s *hubTagRepoStub) DeleteSkill(int) error { return nil } +func (s *hubTagRepoStub) GetBlobByContentHash(string) (*models.SkillBlob, error) { return nil, nil } +func (s *hubTagRepoStub) GetBlobByID(int) (*models.SkillBlob, error) { return nil, nil } +func (s *hubTagRepoStub) CreateBlob(*models.SkillBlob) error { return nil } +func (s *hubTagRepoStub) UpdateBlob(*models.SkillBlob) error { return nil } +func (s *hubTagRepoStub) ListVersionsBySkillID(int) ([]models.SkillVersion, error) { return nil, nil } +func (s *hubTagRepoStub) GetVersionByID(int) (*models.SkillVersion, error) { return nil, nil } +func (s *hubTagRepoStub) GetVersionBySkillAndBlob(int, int) (*models.SkillVersion, error) { + return nil, nil +} +func (s *hubTagRepoStub) GetLatestVersionBySkillID(int) (*models.SkillVersion, error) { + return nil, nil +} +func (s *hubTagRepoStub) CreateVersion(*models.SkillVersion) error { return nil } +func (s *hubTagRepoStub) UpdateVersion(*models.SkillVersion) error { return nil } +func (s *hubTagRepoStub) ListInstanceSkills(int) ([]models.InstanceSkill, error) { + return nil, nil +} +func (s *hubTagRepoStub) ListActiveInstanceSkillsBySkillID(int) ([]models.InstanceSkill, error) { + return nil, nil +} +func (s *hubTagRepoStub) GetInstanceSkill(int, int) (*models.InstanceSkill, error) { return nil, nil } +func (s *hubTagRepoStub) UpsertInstanceSkill(*models.InstanceSkill) error { return nil } +func (s *hubTagRepoStub) MarkInstanceSkillRemoved(int, int, time.Time) error { return nil } +func (s *hubTagRepoStub) MarkInstanceSkillRemovedBySkillKey(int, string, time.Time) error { + return nil +} +func (s *hubTagRepoStub) MarkInstanceSkillsRemovedByWorkspacePath(int, string, time.Time) error { + return nil +} +func (s *hubTagRepoStub) MarkMissingInstanceSkills(int, []int, time.Time) error { return nil } +func (s *hubTagRepoStub) CreateScanResult(*models.SkillScanResult) error { return nil } +func (s *hubTagRepoStub) GetScanResultByID(int) (*models.SkillScanResult, error) { return nil, nil } +func (s *hubTagRepoStub) ListScanResultsByBlobID(int) ([]models.SkillScanResult, error) { + return nil, nil +} +func (s *hubTagRepoStub) GetLatestScanResultByBlobID(int) (*models.SkillScanResult, error) { + return nil, nil +} +func (s *hubTagRepoStub) GetLatestScanResultBySkillID(int) (*models.SkillScanResult, error) { + return nil, nil +} +func (s *hubTagRepoStub) ListHubTags(bool) ([]models.SkillHubTag, error) { return nil, nil } +func (s *hubTagRepoStub) GetHubTagByID(id int) (*models.SkillHubTag, error) { + if tag, ok := s.tags[id]; ok { + return tag, nil + } + return nil, nil +} +func (s *hubTagRepoStub) ListHubTagsBySkillID(int) ([]models.SkillHubTag, error) { return nil, nil } +func (s *hubTagRepoStub) ReplaceSkillTagAssignments(int, []int) error { return nil } +func (s *hubTagRepoStub) ListPublicHubSkills() ([]models.Skill, error) { return nil, nil } +func (s *hubTagRepoStub) ListSkillsForHubAdmin() ([]models.Skill, error) { return nil, nil } diff --git a/backend/internal/services/skill_hub_service.go b/backend/internal/services/skill_hub_service.go new file mode 100644 index 0000000..bca729e --- /dev/null +++ b/backend/internal/services/skill_hub_service.go @@ -0,0 +1,808 @@ +package services + +import ( + "context" + "fmt" + "mime/multipart" + "strings" + "time" + + "clawreef/internal/models" +) + +const ( + skillVisibilityPrivate = "private" + skillVisibilityPublic = "public" +) + +type SkillHubTagPayload struct { + ID int `json:"id"` + TagKey string `json:"tag_key"` + Name string `json:"name"` + Description *string `json:"description,omitempty"` + SortOrder int `json:"sort_order"` + AdminOnly bool `json:"admin_only"` +} + +type SkillHubCatalogQuery struct { + TagKeys []string + Search string + Page int + PageSize int +} + +type SkillHubCatalogResponse struct { + Items []SkillPayload `json:"items"` + Total int `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` + TotalPages int `json:"total_pages"` +} + +type PublishSkillHubRequest struct { + TagIDs []int `json:"tag_ids" binding:"required,min=1"` +} + +type UpdateSkillHubTagsRequest struct { + TagIDs []int `json:"tag_ids" binding:"required,min=1"` +} + +type InstallHubSkillRequest struct { + InstanceID int `json:"instance_id" binding:"required,min=1"` +} + +func isAdminRole(role string) bool { + return strings.EqualFold(strings.TrimSpace(role), "admin") +} + +func (s *skillService) skillBlobForPublish(skill *models.Skill) (*models.SkillBlob, error) { + if skill == nil || skill.CurrentVersionID == nil { + return nil, fmt.Errorf("skill has no version") + } + version, err := s.repo.GetVersionByID(*skill.CurrentVersionID) + if err != nil { + return nil, err + } + if version == nil { + return nil, fmt.Errorf("skill has no version") + } + blob, err := s.repo.GetBlobByID(version.BlobID) + if err != nil { + return nil, err + } + if blob == nil { + return nil, fmt.Errorf("skill blob not found") + } + return blob, nil +} + +func isHubPublishableBlob(blob *models.SkillBlob) bool { + if blob == nil { + return false + } + if !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") { + return false + } + risk := strings.ToLower(strings.TrimSpace(blob.RiskLevel)) + if risk != skillRiskNone && risk != skillRiskLow { + return false + } + if strings.TrimSpace(blob.ObjectKey) == "" { + return false + } + return true +} + +func (s *skillService) isHubPublishable(skill *models.Skill, blob *models.SkillBlob) bool { + if skill == nil || blob == nil || isDeletedSkill(skill) { + return false + } + if !isUserManagedSkill(*skill) && !strings.EqualFold(strings.TrimSpace(skill.SourceType), skillSourceDiscovered) { + return false + } + if !strings.EqualFold(strings.TrimSpace(skill.Status), "active") { + return false + } + return isHubPublishableBlob(blob) +} + +func (s *skillService) CanDownloadSkill(actorUserID int, actorRole string, skill *models.Skill) bool { + if skill == nil || !isUserManagedSkill(*skill) { + return false + } + return s.CanViewSkill(actorUserID, actorRole, skill) +} + +func (s *skillService) CanViewSkill(actorUserID int, actorRole string, skill *models.Skill) bool { + if skill == nil || isDeletedSkill(skill) { + return false + } + if isAdminRole(actorRole) { + return isUserManagedSkill(*skill) || strings.EqualFold(skill.SourceType, skillSourceDiscovered) + } + if skill.UserID == actorUserID { + return isUserManagedSkill(*skill) || strings.EqualFold(skill.SourceType, skillSourceDiscovered) + } + return isUserManagedSkill(*skill) && strings.EqualFold(strings.TrimSpace(skill.Visibility), skillVisibilityPublic) +} + +func (s *skillService) CanAttachSkill(actorUserID int, actorRole string, skill *models.Skill, instance *models.Instance) bool { + if skill == nil || instance == nil { + return false + } + if !isUserManagedSkill(*skill) { + return false + } + if !strings.EqualFold(strings.TrimSpace(skill.Status), "active") { + return false + } + if skill.RiskLevel == skillRiskMedium || skill.RiskLevel == skillRiskHigh { + return false + } + if isAdminRole(actorRole) { + return true + } + if instance.UserID != actorUserID { + return false + } + if skill.UserID == actorUserID { + return true + } + return strings.EqualFold(strings.TrimSpace(skill.Visibility), skillVisibilityPublic) +} + +func (s *skillService) hubTagsToPayload(tags []models.SkillHubTag) []SkillHubTagPayload { + result := make([]SkillHubTagPayload, 0, len(tags)) + for _, tag := range tags { + result = append(result, SkillHubTagPayload{ + ID: tag.ID, + TagKey: tag.TagKey, + Name: tag.Name, + Description: tag.Description, + SortOrder: tag.SortOrder, + AdminOnly: tag.AdminOnly, + }) + } + return result +} + +func (s *skillService) liteInstanceForSkill(skillID int) *models.Instance { + if s == nil || s.repo == nil || s.instanceRepo == nil || skillID <= 0 { + return nil + } + items, err := s.repo.ListActiveInstanceSkillsBySkillID(skillID) + if err != nil || len(items) == 0 { + return nil + } + for _, item := range items { + instance, err := s.instanceRepo.GetByID(item.InstanceID) + if err != nil || instance == nil { + continue + } + if isLiteRuntimeInstance(instance) { + return instance + } + } + return nil +} + +func (s *skillService) enrichSkillPayload(payload *SkillPayload, skill models.Skill, instance *models.Instance) error { + if instance == nil { + instance = s.liteInstanceForSkill(skill.ID) + } + tags, err := s.repo.ListHubTagsBySkillID(skill.ID) + if err != nil { + return err + } + payload.Visibility = skill.Visibility + if strings.TrimSpace(payload.Visibility) == "" { + payload.Visibility = skillVisibilityPrivate + } + payload.PublishedAt = skill.PublishedAt + payload.PublishedBy = skill.PublishedBy + payload.Tags = s.hubTagsToPayload(tags) + + blob, blobErr := s.skillBlobForPublish(&skill) + if blobErr == nil { + payload.Publishable = s.isHubPublishable(&skill, blob) + payload.ScanStatus = blob.ScanStatus + } else { + payload.Publishable = false + } + skipAgentCollectFailure := instance != nil && isLiteRuntimeInstance(instance) + payload.PublishBlockedReason = s.publishBlockedReasonForSkill(&skill, blob, blobErr, payload.Publishable, skipAgentCollectFailure) + if s.materializeService != nil && blobErr == nil && blob != nil && strings.TrimSpace(blob.ObjectKey) == "" { + if status, materializeErr := s.materializeService.GetObservedStatus(skill.ID, blob); status != nil { + payload.PackageMaterializeStatus = status + payload.PackageMaterializeError = materializeErr + } + } + if collectErr := s.resolvePackageCollectError(skill.ID, blob, blobErr, skipAgentCollectFailure); collectErr != nil { + payload.PackageCollectError = collectErr + } + + if s.userRepo != nil { + owner, err := s.userRepo.GetByID(skill.UserID) + if err != nil { + return err + } + if owner != nil { + payload.OwnerUsername = &owner.Username + } + } + return nil +} + +func truncateCollectError(value string, maxLen int) string { + value = strings.TrimSpace(value) + if maxLen <= 0 || len(value) <= maxLen { + return value + } + return value[:maxLen] + "..." +} + +func (s *skillService) resolvePackageCollectError(skillID int, blob *models.SkillBlob, blobErr error, skipAgentCollectFailure bool) *string { + if blobErr != nil || blob == nil || strings.TrimSpace(blob.ObjectKey) != "" { + return nil + } + if s.materializeService != nil { + if _, materializeErr := s.materializeService.GetObservedStatus(skillID, blob); materializeErr != nil { + return materializeErr + } + job, err := s.materializeService.FindLatestBySkillID(skillID) + if err == nil && job != nil && job.LastError != nil && strings.TrimSpace(*job.LastError) != "" { + summary := truncateCollectError(*job.LastError, 512) + if summary != "" { + return &summary + } + } + } + if skipAgentCollectFailure { + return nil + } + cmd, err := s.latestCollectPackageFailure(skillID) + if err != nil || cmd == nil { + return nil + } + if cmd.ErrorMessage == nil { + return nil + } + summary := truncateCollectError(*cmd.ErrorMessage, 512) + if summary == "" { + return nil + } + return &summary +} + +func (s *skillService) latestCollectPackageFailure(skillID int) (*models.InstanceCommand, error) { + if s.commandRepo == nil { + return nil, nil + } + return s.commandRepo.FindLatestFailedCollectSkillPackage(formatExternalSkillID(skillID)) +} + +func (s *skillService) publishBlockedReasonForSkill(skill *models.Skill, blob *models.SkillBlob, blobErr error, publishable bool, skipAgentCollectFailure bool) *string { + if publishable || skill == nil { + return nil + } + reason := func(value string) *string { + return &value + } + if isDeletedSkill(skill) { + return reason("skill_deleted") + } + if !strings.EqualFold(strings.TrimSpace(skill.Status), skillStatusActive) { + return reason("skill_inactive") + } + if blobErr != nil || blob == nil { + return reason("skill_package_pending") + } + if strings.TrimSpace(blob.ObjectKey) == "" { + if s.materializeService != nil { + if job, err := s.materializeService.FindLatestBySkillID(skill.ID); err == nil && job != nil { + if blocked := materializeBlockedReason(job); blocked != nil { + return blocked + } + return reason("skill_package_pending") + } + } + if skipAgentCollectFailure { + return reason("skill_package_pending") + } + if cmd, err := s.latestCollectPackageFailure(skill.ID); err == nil && cmd != nil { + return reason("skill_package_collect_failed") + } + return reason("skill_package_pending") + } + if strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "failed") { + return reason("skill_scan_failed") + } + if !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") { + return reason("skill_not_scanned") + } + risk := strings.ToLower(strings.TrimSpace(blob.RiskLevel)) + if risk != skillRiskNone && risk != skillRiskLow { + return reason("skill_risk_blocked") + } + return nil +} + +func (s *skillService) validateHubTagSelection(actorRole string, tagIDs []int) error { + if len(tagIDs) == 0 { + return fmt.Errorf("skill_tags_required") + } + hasPublicTag := false + for _, tagID := range tagIDs { + tag, err := s.repo.GetHubTagByID(tagID) + if err != nil { + return err + } + if tag == nil { + return fmt.Errorf("skill hub tag not found") + } + if tag.AdminOnly && !isAdminRole(actorRole) { + return fmt.Errorf("access denied") + } + if !tag.AdminOnly { + hasPublicTag = true + } + } + if !hasPublicTag { + return fmt.Errorf("skill_tags_required") + } + return nil +} + +func (s *skillService) ListHubTags(actorRole string) ([]SkillHubTagPayload, error) { + tags, err := s.repo.ListHubTags(isAdminRole(actorRole)) + if err != nil { + return nil, err + } + return s.hubTagsToPayload(tags), nil +} + +func (s *skillService) ListHubCatalog(_ int, _ string, query SkillHubCatalogQuery) (*SkillHubCatalogResponse, error) { + if query.Page <= 0 { + query.Page = 1 + } + if query.PageSize <= 0 { + query.PageSize = 20 + } + if query.PageSize > 1000 { + query.PageSize = 1000 + } + + items, err := s.repo.ListPublicHubSkills() + if err != nil { + return nil, err + } + + tagKeySet := map[string]struct{}{} + for _, key := range query.TagKeys { + key = strings.TrimSpace(key) + if key != "" { + tagKeySet[key] = struct{}{} + } + } + search := strings.ToLower(strings.TrimSpace(query.Search)) + + filtered := make([]SkillPayload, 0, len(items)) + for _, item := range items { + blob, blobErr := s.skillBlobForPublish(&item) + if blobErr != nil || !s.isHubPublishable(&item, blob) { + continue + } + if len(tagKeySet) > 0 { + tags, err := s.repo.ListHubTagsBySkillID(item.ID) + if err != nil { + return nil, err + } + matched := false + for _, tag := range tags { + if _, ok := tagKeySet[tag.TagKey]; ok { + matched = true + break + } + } + if !matched { + continue + } + } + if search != "" { + haystack := strings.ToLower(strings.Join([]string{item.Name, item.SkillKey, derefString(item.Description)}, " ")) + if !strings.Contains(haystack, search) { + continue + } + } + payload, err := s.toSkillPayload(item) + if err != nil { + return nil, err + } + if err := s.enrichSkillPayload(payload, item, nil); err != nil { + return nil, err + } + filtered = append(filtered, *payload) + } + + total := len(filtered) + start := (query.Page - 1) * query.PageSize + if start > total { + start = total + } + end := start + query.PageSize + if end > total { + end = total + } + pageItems := filtered[start:end] + totalPages := total / query.PageSize + if total%query.PageSize != 0 { + totalPages++ + } + if totalPages == 0 { + totalPages = 1 + } + + return &SkillHubCatalogResponse{ + Items: pageItems, + Total: total, + Page: query.Page, + PageSize: query.PageSize, + TotalPages: totalPages, + }, nil +} + +func (s *skillService) ListMyHubSkills(userID int) ([]SkillPayload, error) { + items, err := s.repo.ListSkillsByUser(userID) + if err != nil { + return nil, err + } + filtered := make([]models.Skill, 0, len(items)) + for _, item := range items { + if isDeletedSkill(&item) { + continue + } + if isUserManagedSkill(item) { + filtered = append(filtered, item) + } + } + result := make([]SkillPayload, 0, len(filtered)) + for _, item := range filtered { + payload, err := s.toSkillPayload(item) + if err != nil { + return nil, err + } + if err := s.enrichSkillPayload(payload, item, nil); err != nil { + return nil, err + } + result = append(result, *payload) + } + return result, nil +} + +func (s *skillService) ListAllHubSkillsAdmin() ([]SkillPayload, error) { + items, err := s.repo.ListSkillsForHubAdmin() + if err != nil { + return nil, err + } + result := make([]SkillPayload, 0, len(items)) + for _, item := range items { + if isDeletedSkill(&item) { + continue + } + payload, err := s.toSkillPayload(item) + if err != nil { + return nil, err + } + if err := s.enrichSkillPayload(payload, item, nil); err != nil { + return nil, err + } + result = append(result, *payload) + } + return result, nil +} + +func (s *skillService) GetSkillHubDetail(actorUserID int, actorRole string, skillID int) (*SkillPayload, error) { + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil || !s.CanViewSkill(actorUserID, actorRole, skill) { + return nil, fmt.Errorf("skill not found") + } + payload, err := s.toSkillPayload(*skill) + if err != nil { + return nil, err + } + if err := s.enrichSkillPayload(payload, *skill, nil); err != nil { + return nil, err + } + return payload, nil +} + +func (s *skillService) PublishToHub(actorUserID int, actorRole string, skillID int, tagIDs []int) (*SkillPayload, error) { + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil || isDeletedSkill(skill) { + return nil, fmt.Errorf("skill not found") + } + if skill.UserID != actorUserID && !isAdminRole(actorRole) { + return nil, fmt.Errorf("skill not found") + } + if err := s.validateHubTagSelection(actorRole, tagIDs); err != nil { + return nil, err + } + blob, err := s.skillBlobForPublish(skill) + if err != nil { + return nil, err + } + if strings.TrimSpace(blob.ObjectKey) == "" { + return nil, fmt.Errorf("skill_package_pending") + } + if !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") { + return nil, fmt.Errorf("skill_not_scanned") + } + if !isHubPublishableBlob(blob) { + return nil, fmt.Errorf("skill_risk_blocked") + } + if strings.EqualFold(skill.SourceType, skillSourceDiscovered) { + skill.SourceType = skillSourceUploaded + } + if err := s.repo.ReplaceSkillTagAssignments(skillID, tagIDs); err != nil { + return nil, err + } + now := time.Now().UTC() + skill.Visibility = skillVisibilityPublic + skill.PublishedAt = &now + skill.PublishedBy = &actorUserID + skill.UpdatedAt = now + if err := s.repo.UpdateSkill(skill); err != nil { + return nil, err + } + return s.GetSkillHubDetail(actorUserID, actorRole, skillID) +} + +func (s *skillService) UnpublishFromHub(actorUserID int, actorRole string, skillID int) (*SkillPayload, error) { + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil { + return nil, fmt.Errorf("skill not found") + } + if skill.UserID != actorUserID && !isAdminRole(actorRole) { + return nil, fmt.Errorf("skill not found") + } + skill.Visibility = skillVisibilityPrivate + skill.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateSkill(skill); err != nil { + return nil, err + } + return s.GetSkillHubDetail(actorUserID, actorRole, skillID) +} + +func (s *skillService) UpdateHubTags(actorUserID int, actorRole string, skillID int, tagIDs []int) (*SkillPayload, error) { + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil { + return nil, fmt.Errorf("skill not found") + } + if skill.UserID != actorUserID && !isAdminRole(actorRole) { + return nil, fmt.Errorf("skill not found") + } + if !strings.EqualFold(strings.TrimSpace(skill.Visibility), skillVisibilityPublic) { + return nil, fmt.Errorf("skill is not published to hub") + } + if err := s.validateHubTagSelection(actorRole, tagIDs); err != nil { + return nil, err + } + if err := s.repo.ReplaceSkillTagAssignments(skillID, tagIDs); err != nil { + return nil, err + } + skill.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateSkill(skill); err != nil { + return nil, err + } + return s.GetSkillHubDetail(actorUserID, actorRole, skillID) +} + +func (s *skillService) InstallHubSkill(actorUserID int, actorRole string, skillID, instanceID int) (*InstanceSkillPayload, error) { + return s.AttachSkillToInstance(actorUserID, actorRole, instanceID, skillID) +} + +func (s *skillService) ImportInstanceSkillToLibrary(actorUserID int, actorRole string, instanceID, skillID int) (*SkillPayload, error) { + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, err + } + if instance == nil { + return nil, fmt.Errorf("instance not found") + } + if !isAdminRole(actorRole) && instance.UserID != actorUserID { + return nil, fmt.Errorf("access denied") + } + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil || skill.UserID != instance.UserID || isDeletedSkill(skill) { + return nil, fmt.Errorf("skill not found") + } + instanceSkill, err := s.repo.GetInstanceSkill(instanceID, skillID) + if err != nil { + return nil, err + } + if instanceSkill == nil || instanceSkill.Status == "removed" { + return nil, fmt.Errorf("skill not found on instance") + } + if isUserManagedSkill(*skill) { + blob, blobErr := s.skillBlobForPublish(skill) + if blobErr == nil && strings.TrimSpace(blob.ObjectKey) != "" && strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") { + return s.GetSkillHubDetail(actorUserID, actorRole, skillID) + } + } + if err := s.requestSkillPackageCollection(instanceID, skill, instanceSkill, fmt.Sprintf("import-%d-%d", instanceID, skillID)); err != nil { + return nil, err + } + blob, err := s.skillBlobForPublish(skill) + if err != nil { + return nil, err + } + content, err := s.storage.GetObject(context.Background(), blob.ObjectKey) + if err != nil { + return nil, err + } + if err := s.ensureBlobObject(context.Background(), blob, content); err != nil { + return nil, err + } + if blob.LastScanResultID == nil || !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") { + if err := s.recordScanFromStoredBlob(blob); err != nil { + return nil, err + } + } + skill, err = s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil { + return nil, fmt.Errorf("skill not found") + } + blob, err = s.repo.GetBlobByID(blob.ID) + if err != nil { + return nil, err + } + if blob != nil { + skill.RiskLevel = blob.RiskLevel + skill.LastScannedAt = blob.LastScannedAt + skill.LastScanResultID = blob.LastScanResultID + } + if err := s.promoteSkillToUploadedLibrary(skill); err != nil { + return nil, err + } + return s.GetSkillHubDetail(actorUserID, actorRole, skillID) +} + +func (s *skillService) RetrySkillPackageCollection(actorUserID int, actorRole string, instanceID, skillID int) error { + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return err + } + if instance == nil { + return fmt.Errorf("instance not found") + } + if !isAdminRole(actorRole) && instance.UserID != actorUserID { + return fmt.Errorf("access denied") + } + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return err + } + if skill == nil || skill.UserID != instance.UserID || isDeletedSkill(skill) { + return fmt.Errorf("skill not found") + } + instanceSkill, err := s.repo.GetInstanceSkill(instanceID, skillID) + if err != nil { + return err + } + if instanceSkill == nil || instanceSkill.Status == "removed" { + return fmt.Errorf("skill not found on instance") + } + blob, blobErr := s.skillBlobForPublish(skill) + if blobErr == nil && strings.TrimSpace(blob.ObjectKey) != "" && strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") { + return nil + } + if isLiteRuntimeInstance(instance) && s.materializeService != nil { + if job, findErr := s.materializeService.FindLatestBySkillID(skillID); findErr == nil && job != nil && strings.EqualFold(strings.TrimSpace(job.Status), MaterializeJobStatusFailed) { + _ = s.materializeService.RetryJob(skillID) + } + } + return s.requestSkillPackageCollection(instanceID, skill, instanceSkill, fmt.Sprintf("retry-%d-%d-%d", instanceID, skillID, time.Now().Unix())) +} + +func (s *skillService) PublishFromInstance(actorUserID int, actorRole string, instanceID, skillID int, tagIDs []int) (*SkillPayload, error) { + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, err + } + if instance == nil { + return nil, fmt.Errorf("instance not found") + } + if !isAdminRole(actorRole) && instance.UserID != actorUserID { + return nil, fmt.Errorf("access denied") + } + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil || skill.UserID != instance.UserID { + return nil, fmt.Errorf("skill not found") + } + if !isUserManagedSkill(*skill) { + return nil, fmt.Errorf("skill_not_in_library") + } + instanceSkill, err := s.repo.GetInstanceSkill(instanceID, skillID) + if err != nil { + return nil, err + } + if instanceSkill == nil || instanceSkill.Status == "removed" { + return nil, fmt.Errorf("skill not found on instance") + } + if err := s.requestSkillPackageCollection(instanceID, skill, instanceSkill, fmt.Sprintf("publish-%d-%d", instanceID, skillID)); err != nil { + return nil, err + } + return s.PublishToHub(actorUserID, actorRole, skillID, tagIDs) +} + +func (s *skillService) ListAttachableSkills(actorUserID int, actorRole string) ([]SkillPayload, error) { + result := make([]SkillPayload, 0) + seen := map[int]struct{}{} + + mine, err := s.ListMyHubSkills(actorUserID) + if err != nil { + return nil, err + } + for _, item := range mine { + if strings.EqualFold(item.SourceType, skillSourceDiscovered) { + continue + } + if item.Status != "active" || item.RiskLevel == skillRiskMedium || item.RiskLevel == skillRiskHigh { + continue + } + if _, ok := seen[item.ID]; ok { + continue + } + seen[item.ID] = struct{}{} + result = append(result, item) + } + + catalog, err := s.ListHubCatalog(actorUserID, actorRole, SkillHubCatalogQuery{Page: 1, PageSize: 1000}) + if err != nil { + return nil, err + } + for _, item := range catalog.Items { + if item.UserID == actorUserID { + continue + } + if _, ok := seen[item.ID]; ok { + continue + } + seen[item.ID] = struct{}{} + result = append(result, item) + } + return result, nil +} + +func (s *skillService) ImportHubArchive(ctx context.Context, userID int, fileHeader *multipart.FileHeader) ([]SkillPayload, error) { + items, err := s.ImportHubArchiveWithDecisions(ctx, userID, fileHeader, nil) + if err != nil { + return nil, err + } + results := make([]SkillPayload, 0, len(items)) + for _, item := range items { + results = append(results, item.Skill) + } + return results, nil +} diff --git a/backend/internal/services/skill_hub_service_test.go b/backend/internal/services/skill_hub_service_test.go new file mode 100644 index 0000000..b0f3d06 --- /dev/null +++ b/backend/internal/services/skill_hub_service_test.go @@ -0,0 +1,957 @@ +package services + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "clawreef/internal/models" +) + +func TestIsHubPublishableBlob(t *testing.T) { + tests := []struct { + name string + blob *models.SkillBlob + want bool + }{ + { + name: "completed none risk with object key", + blob: &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "user/demo/hash.zip"}, + want: true, + }, + { + name: "completed low risk", + blob: &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskLow, ObjectKey: "key"}, + want: true, + }, + { + name: "medium risk blocked", + blob: &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskMedium, ObjectKey: "key"}, + want: false, + }, + { + name: "pending scan blocked", + blob: &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskNone, ObjectKey: "key"}, + want: false, + }, + { + name: "missing object key blocked", + blob: &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: ""}, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := isHubPublishableBlob(tc.blob); got != tc.want { + t.Fatalf("isHubPublishableBlob() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestListMyHubSkillsExcludesDiscoveredSkills(t *testing.T) { + svc := &skillService{ + repo: &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: {ID: 1, UserID: 1, SkillKey: "dogfood", Name: "dogfood", Status: skillStatusActive, SourceType: skillSourceUploaded}, + 2: {ID: 2, UserID: 1, SkillKey: "software-development-spike", Name: "software-development/spike", Status: skillStatusActive, SourceType: skillSourceDiscovered}, + }, + }, + } + items, err := svc.ListMyHubSkills(1) + if err != nil { + t.Fatalf("ListMyHubSkills() error = %v", err) + } + if len(items) != 1 { + t.Fatalf("items = %d, want 1 uploaded skill only", len(items)) + } + if items[0].SkillKey != "dogfood" { + t.Fatalf("SkillKey = %q, want dogfood", items[0].SkillKey) + } +} + +func TestCanAttachSkillRules(t *testing.T) { + svc := &skillService{} + privateSkill := &models.Skill{UserID: 1, SourceType: skillSourceUploaded, Status: "active", Visibility: skillVisibilityPrivate, RiskLevel: skillRiskLow} + publicSkill := &models.Skill{UserID: 2, SourceType: skillSourceUploaded, Status: "active", Visibility: skillVisibilityPublic, RiskLevel: skillRiskLow} + ownInstance := &models.Instance{UserID: 1} + otherInstance := &models.Instance{UserID: 3} + + if !svc.CanAttachSkill(1, "user", privateSkill, ownInstance) { + t.Fatal("owner should attach private skill to own instance") + } + if svc.CanAttachSkill(3, "user", privateSkill, otherInstance) { + t.Fatal("other user must not attach private skill") + } + if !svc.CanAttachSkill(3, "user", publicSkill, otherInstance) { + t.Fatal("user should attach public skill to own instance") + } + if svc.CanAttachSkill(3, "user", publicSkill, ownInstance) { + t.Fatal("user must not attach public skill to someone else's instance") + } + if !svc.CanAttachSkill(99, "admin", privateSkill, otherInstance) { + t.Fatal("admin should attach any skill to any instance") + } +} + +func TestCanViewSkillRules(t *testing.T) { + svc := &skillService{} + privateSkill := &models.Skill{UserID: 1, SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate} + publicSkill := &models.Skill{UserID: 2, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic} + + if !svc.CanViewSkill(1, "user", privateSkill) { + t.Fatal("owner should view private skill") + } + if svc.CanViewSkill(3, "user", privateSkill) { + t.Fatal("other user must not view private skill") + } + if !svc.CanViewSkill(3, "user", publicSkill) { + t.Fatal("user should view public skill") + } + if !svc.CanViewSkill(99, "admin", privateSkill) { + t.Fatal("admin should view private skill") + } +} + +func TestCanDownloadSkillRules(t *testing.T) { + svc := &skillService{} + privateSkill := &models.Skill{UserID: 1, SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate} + publicSkill := &models.Skill{UserID: 2, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic} + discoveredSkill := &models.Skill{UserID: 1, SourceType: skillSourceDiscovered, Visibility: skillVisibilityPublic} + + if !svc.CanDownloadSkill(1, "user", privateSkill) { + t.Fatal("owner should download private uploaded skill") + } + if svc.CanDownloadSkill(3, "user", privateSkill) { + t.Fatal("other user must not download private skill") + } + if !svc.CanDownloadSkill(3, "user", publicSkill) { + t.Fatal("user should download public skill") + } + if svc.CanDownloadSkill(1, "user", discoveredSkill) { + t.Fatal("discovered skill is not user-managed and must not download") + } +} + +func TestIsHubPublishableSkill(t *testing.T) { + svc := &skillService{} + blob := &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "user/demo/hash.zip"} + activeSkill := &models.Skill{SourceType: skillSourceUploaded, Status: "active"} + inactiveSkill := &models.Skill{SourceType: skillSourceUploaded, Status: "inactive"} + + if !svc.isHubPublishable(activeSkill, blob) { + t.Fatal("active uploaded skill with clean blob should be publishable") + } + if svc.isHubPublishable(inactiveSkill, blob) { + t.Fatal("inactive skill must not be publishable") + } + if svc.isHubPublishable(activeSkill, &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskNone, ObjectKey: "key"}) { + t.Fatal("pending scan blob must block publish") + } +} + +func TestValidateHubTagSelection(t *testing.T) { + svc := &skillService{repo: &hubTagRepoStub{ + tags: map[int]*models.SkillHubTag{ + 1: {ID: 1, TagKey: "coding", Name: "Coding", AdminOnly: false}, + 2: {ID: 2, TagKey: "featured", Name: "Featured", AdminOnly: true}, + }, + }} + + if err := svc.validateHubTagSelection("user", nil); err == nil || err.Error() != "skill_tags_required" { + t.Fatalf("empty tag list should require tags, got %v", err) + } + if err := svc.validateHubTagSelection("user", []int{2}); err == nil || err.Error() != "access denied" { + t.Fatalf("user must not select admin-only tag alone, got %v", err) + } + if err := svc.validateHubTagSelection("user", []int{1}); err != nil { + t.Fatalf("user with public tag should pass, got %v", err) + } + if err := svc.validateHubTagSelection("admin", []int{2}); err == nil || err.Error() != "skill_tags_required" { + t.Fatalf("admin-only tag alone should still require a public tag, got %v", err) + } + if err := svc.validateHubTagSelection("admin", []int{1, 2}); err != nil { + t.Fatalf("admin with mixed tags should pass, got %v", err) + } +} + +func newPublishTestStub(blob *models.SkillBlob) (*skillService, *skillRepoStub) { + versionID := 10 + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, + SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, BlobID: blobID}}, + blobs: map[int]*models.SkillBlob{blobID: blob}, + tags: map[int]*models.SkillHubTag{ + 1: {ID: 1, TagKey: "coding", Name: "Coding", AdminOnly: false}, + }, + tagAssignments: map[int][]int{}, + } + return &skillService{repo: stub}, stub +} + +func TestDeleteSkillSoftPreservesInstanceSkills(t *testing.T) { + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: {ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic}, + }, + tagAssignments: map[int][]int{1: {1}}, + instanceSkillsBySkillID: map[int][]models.InstanceSkill{ + 1: {{ID: 99, InstanceID: 2, SkillID: 1, Status: "active"}}, + }, + } + svc := &skillService{repo: stub} + if err := svc.DeleteSkill(1, "user", 1); err != nil { + t.Fatalf("DeleteSkill() error = %v", err) + } + if stub.hardDeleteCalled { + t.Fatal("DeleteSkill must not hard-delete skill row") + } + if stub.skills[1].Status != skillStatusDeleted { + t.Fatalf("expected soft-deleted status, got %q", stub.skills[1].Status) + } + if len(stub.tagAssignments[1]) != 0 { + t.Fatalf("expected tag assignments cleared, got %v", stub.tagAssignments[1]) + } + if len(stub.instanceSkillsBySkillID[1]) != 1 { + t.Fatal("instance_skills records must remain after soft delete") + } +} + +func TestCanViewSkillDeletedHidden(t *testing.T) { + svc := &skillService{} + deletedSkill := &models.Skill{UserID: 1, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic, Status: skillStatusDeleted} + if svc.CanViewSkill(3, "user", deletedSkill) { + t.Fatal("deleted public skill must not be viewable") + } +} + +func TestPublishToHubRejectsPendingScan(t *testing.T) { + svc, _ := newPublishTestStub(&models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskNone, ObjectKey: "key.zip"}) + _, err := svc.PublishToHub(1, "user", 1, []int{1}) + if err == nil || err.Error() != "skill_not_scanned" { + t.Fatalf("expected skill_not_scanned, got %v", err) + } +} + +func TestPublishToHubRejectsMediumRisk(t *testing.T) { + svc, _ := newPublishTestStub(&models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskMedium, ObjectKey: "key.zip"}) + _, err := svc.PublishToHub(1, "user", 1, []int{1}) + if err == nil || err.Error() != "skill_risk_blocked" { + t.Fatalf("expected skill_risk_blocked, got %v", err) + } +} + +func TestPublishToHubRejectsEmptyTags(t *testing.T) { + svc, _ := newPublishTestStub(&models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"}) + _, err := svc.PublishToHub(1, "user", 1, nil) + if err == nil || err.Error() != "skill_tags_required" { + t.Fatalf("expected skill_tags_required, got %v", err) + } +} + +func TestPublishToHubAllowsAdminForOtherUsersSkill(t *testing.T) { + svc, _ := newPublishTestStub(&models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"}) + item, err := svc.PublishToHub(99, "admin", 1, []int{1}) + if err != nil { + t.Fatalf("PublishToHub() error = %v", err) + } + if item == nil || !strings.EqualFold(item.Visibility, skillVisibilityPublic) { + t.Fatalf("expected admin publish to succeed, got %#v", item) + } +} + +func TestUnpublishAllowsAdminForOtherUsersSkill(t *testing.T) { + svc, stub := newPublishTestStub(&models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"}) + stub.skills[1].Visibility = skillVisibilityPublic + item, err := svc.UnpublishFromHub(99, "admin", 1) + if err != nil { + t.Fatalf("UnpublishFromHub() error = %v", err) + } + if item == nil || !strings.EqualFold(item.Visibility, skillVisibilityPrivate) { + t.Fatalf("expected admin unpublish to succeed, got %#v", item) + } +} + +func TestUnpublishRemovesFromPublicCatalog(t *testing.T) { + versionID := 10 + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, + SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{10: {ID: 10, BlobID: blobID}}, + blobs: map[int]*models.SkillBlob{blobID: {ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"}}, + } + svc := &skillService{repo: stub} + if _, err := svc.UnpublishFromHub(1, "user", 1); err != nil { + t.Fatalf("UnpublishFromHub() error = %v", err) + } + if stub.skills[1].Visibility != skillVisibilityPrivate { + t.Fatalf("expected private visibility after unpublish, got %q", stub.skills[1].Visibility) + } + publicSkills, err := stub.ListPublicHubSkills() + if err != nil { + t.Fatalf("ListPublicHubSkills() error = %v", err) + } + if len(publicSkills) != 0 { + t.Fatalf("catalog source should be empty after unpublish, got %d items", len(publicSkills)) + } +} + +func TestListAllHubSkillsAdminExcludesDeleted(t *testing.T) { + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: {ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic}, + 2: {ID: 2, UserID: 1, SkillKey: "demo__deleted_2", Name: "Demo", Status: skillStatusDeleted, SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate}, + }, + tagAssignments: map[int][]int{}, + } + svc := &skillService{repo: stub} + items, err := svc.ListAllHubSkillsAdmin() + if err != nil { + t.Fatalf("ListAllHubSkillsAdmin() error = %v", err) + } + if len(items) != 1 { + t.Fatalf("ListAllHubSkillsAdmin() len = %d, want 1", len(items)) + } + if items[0].ID != 1 { + t.Fatalf("ListAllHubSkillsAdmin() id = %d, want 1", items[0].ID) + } +} + +type importTestInstanceRepo struct { + instances map[int]*models.Instance +} + +func (r *importTestInstanceRepo) Create(*models.Instance) error { panic("not used") } +func (r *importTestInstanceRepo) GetByID(id int) (*models.Instance, error) { + if inst, ok := r.instances[id]; ok { + copy := *inst + return ©, nil + } + return nil, nil +} +func (r *importTestInstanceRepo) GetByAccessToken(string) (*models.Instance, error) { panic("not used") } +func (r *importTestInstanceRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) { + panic("not used") +} +func (r *importTestInstanceRepo) GetAll(int, int) ([]models.Instance, error) { + items := make([]models.Instance, 0, len(r.instances)) + for _, inst := range r.instances { + items = append(items, *inst) + } + return items, nil +} +func (r *importTestInstanceRepo) CountAll() (int, error) { panic("not used") } +func (r *importTestInstanceRepo) GetByUserID(int, int, int) ([]models.Instance, error) { + panic("not used") +} +func (r *importTestInstanceRepo) CountByUserID(int) (int, error) { panic("not used") } +func (r *importTestInstanceRepo) CountActiveByMode(context.Context, string) (int, error) { + panic("not used") +} +func (r *importTestInstanceRepo) ExistsByUserIDAndName(int, string) (bool, error) { + panic("not used") +} +func (r *importTestInstanceRepo) GetAllRunning() ([]models.Instance, error) { panic("not used") } +func (r *importTestInstanceRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) { + panic("not used") +} +func (r *importTestInstanceRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) { + panic("not used") +} +func (r *importTestInstanceRepo) UpdateRuntimeState(context.Context, int, string, int, *string) error { + panic("not used") +} +func (r *importTestInstanceRepo) SetWorkspacePath(context.Context, int, string) error { + panic("not used") +} +func (r *importTestInstanceRepo) UpdateWorkspaceUsage(context.Context, int, int64) error { + panic("not used") +} +func (r *importTestInstanceRepo) Update(*models.Instance) error { panic("not used") } +func (r *importTestInstanceRepo) Delete(int) error { panic("not used") } + +type noopInstanceCommandService struct{} + +func (n *noopInstanceCommandService) Create(int, *int, CreateInstanceCommandRequest) (*InstanceCommandPayload, error) { + return nil, nil +} +func (n *noopInstanceCommandService) GetNextForAgent(*AgentSession) (*AgentCommandEnvelope, error) { + panic("not used") +} +func (n *noopInstanceCommandService) MarkStarted(*AgentSession, int, *time.Time) error { + panic("not used") +} +func (n *noopInstanceCommandService) MarkFinished(*AgentSession, int, AgentCommandFinishRequest) error { + panic("not used") +} +func (n *noopInstanceCommandService) ListByInstanceID(int, int) ([]InstanceCommandPayload, error) { + panic("not used") +} + +func TestPublishFromInstanceRejectsDiscoveredSkill(t *testing.T) { + versionID := 10 + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, + SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}}, + blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"}}, + instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}}, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + _, err := svc.PublishFromInstance(1, "user", 1, 1, []int{1}) + if err == nil || err.Error() != "skill_not_in_library" { + t.Fatalf("expected skill_not_in_library, got %v", err) + } +} + +func TestImportInstanceSkillToLibraryPendingPackage(t *testing.T) { + versionID := 10 + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, + SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}}, + blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}}, + instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}}, + tagAssignments: map[int][]int{}, + tags: map[int]*models.SkillHubTag{ + 1: {ID: 1, TagKey: "coding", Name: "Coding", AdminOnly: false}, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + _, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1) + if err == nil || err.Error() != "skill_package_pending" { + t.Fatalf("expected skill_package_pending, got %v", err) + } +} + +func TestImportInstanceSkillToLibraryRejectsNonOwner(t *testing.T) { + versionID := 10 + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, + SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}}, + blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}}, + instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}}, + tagAssignments: map[int][]int{}, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + _, err := svc.ImportInstanceSkillToLibrary(2, "user", 1, 1) + if err == nil || err.Error() != "access denied" { + t.Fatalf("expected access denied, got %v", err) + } +} + +func TestImportInstanceSkillToLibraryRejectsDeletedSkill(t *testing.T) { + versionID := 10 + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusDeleted, + SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}}, + blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "discovered/1/demo.zip"}}, + instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}}, + tagAssignments: map[int][]int{}, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + _, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1) + if err == nil || err.Error() != "skill not found" { + t.Fatalf("expected skill not found, got %v", err) + } +} + +func TestImportInstanceSkillToLibraryPromotesDiscoveredSkill(t *testing.T) { + versionID := 10 + blobID := 20 + scanResultID := 99 + objectKey := "discovered/1/demo.zip" + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, + SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, SourceType: skillSourceDiscovered}}, + blobs: map[int]*models.SkillBlob{ + blobID: { + ID: blobID, ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: objectKey, + LastScanResultID: &scanResultID, + }, + }, + instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}}, + tagAssignments: map[int][]int{}, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}} + storage := &importTestObjectStorage{objects: map[string][]byte{objectKey: []byte("fake-zip")}} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}, storage: storage} + _, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1) + if err != nil { + t.Fatalf("ImportInstanceSkillToLibrary() error = %v", err) + } + if stub.skills[1].SourceType != skillSourceUploaded { + t.Fatalf("skill source_type = %q, want %q", stub.skills[1].SourceType, skillSourceUploaded) + } + if stub.versions[versionID].SourceType != skillSourceUploaded { + t.Fatalf("version source_type = %q, want %q", stub.versions[versionID].SourceType, skillSourceUploaded) + } +} + +func TestImportInstanceSkillToLibraryLiteMaterializesPackage(t *testing.T) { + versionID := 10 + blobID := 20 + contentHash := "abc123def456789012345678901234" + workspaceDir := "yuanbao" + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "yuanbao", Name: "yuanbao", Status: skillStatusActive, + SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, SourceType: skillSourceDiscovered}}, + blobs: map[int]*models.SkillBlob{ + blobID: {ID: blobID, ContentHash: contentHash, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}, + }, + instanceSkills: []models.InstanceSkill{{ + InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance", + WorkspaceDir: &workspaceDir, + }}, + tagAssignments: map[int][]int{}, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{ + 1: {ID: 1, UserID: 1, InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway}, + }} + storage := &importTestObjectStorage{objects: map[string][]byte{}} + cmdSvc := &capturingInstanceCommandService{} + matSvc := NewSkillPackageMaterializeService( + &materializeJobRepoStub{}, + stub, + importLiteMaterializer{repo: stub, storage: storage, blobID: blobID}, + ) + svc := &skillService{ + repo: stub, instanceRepo: instRepo, commandService: cmdSvc, storage: storage, materializeService: matSvc, + } + _, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1) + if err != nil { + t.Fatalf("ImportInstanceSkillToLibrary() error = %v", err) + } + for _, req := range cmdSvc.created { + if req.CommandType == InstanceCommandTypeCollectSkillPackage { + t.Fatalf("unexpected collect_skill_package command: %#v", req) + } + } + if stub.skills[1].SourceType != skillSourceUploaded { + t.Fatalf("skill source_type = %q, want %q", stub.skills[1].SourceType, skillSourceUploaded) + } + blob := stub.blobs[blobID] + if blob == nil || strings.TrimSpace(blob.ObjectKey) == "" { + t.Fatalf("expected materialized object key, got %#v", blob) + } +} + +func TestImportInstanceSkillToLibraryIdempotentForUploaded(t *testing.T) { + versionID := 10 + blobID := 20 + scanResultID := 99 + objectKey := "user/1/demo.zip" + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, + SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, SourceType: skillSourceUploaded}}, + blobs: map[int]*models.SkillBlob{ + blobID: { + ID: blobID, ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: objectKey, + LastScanResultID: &scanResultID, + }, + }, + instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}}, + tagAssignments: map[int][]int{}, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}} + storage := &importTestObjectStorage{objects: map[string][]byte{objectKey: []byte("fake-zip")}} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}, storage: storage} + payload, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1) + if err != nil { + t.Fatalf("ImportInstanceSkillToLibrary() error = %v", err) + } + if stub.skills[1].SourceType != skillSourceUploaded { + t.Fatalf("skill source_type = %q, want %q", stub.skills[1].SourceType, skillSourceUploaded) + } + if payload == nil || payload.SourceType != skillSourceUploaded { + t.Fatalf("payload source_type = %v, want %q", payload, skillSourceUploaded) + } +} + +func TestSyncAgentSkillsCreatesDiscoveredSkillWithPrivateVisibility(t *testing.T) { + stub := &capturingSkillRepoStub{ + skillRepoStub: skillRepoStub{ + skills: map[int]*models.Skill{}, + blobs: map[int]*models.SkillBlob{}, + versions: map[int]*models.SkillVersion{}, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{ + Skills: []AgentSkillRecord{{ + Identifier: "weather", + ContentMD5: "abc123def456789012345678901234", + Source: "discovered_in_instance", + }}, + }) + if err != nil { + t.Fatalf("SyncAgentSkills() error = %v", err) + } + if len(stub.createdSkills) != 1 { + t.Fatalf("created %d skills, want 1", len(stub.createdSkills)) + } + if stub.createdSkills[0].Visibility != skillVisibilityPrivate { + t.Fatalf("visibility = %q, want %q", stub.createdSkills[0].Visibility, skillVisibilityPrivate) + } +} + +type capturingSkillRepoStub struct { + skillRepoStub + createdSkills []*models.Skill + nextSkillID int + markMissingCalls int + lastMarkMissingActive []int +} + +func (s *capturingSkillRepoStub) CreateSkill(skill *models.Skill) error { + s.nextSkillID++ + skill.ID = s.nextSkillID + copy := *skill + s.createdSkills = append(s.createdSkills, ©) + if s.skills == nil { + s.skills = map[int]*models.Skill{} + } + stored := *skill + s.skills[skill.ID] = &stored + return nil +} + +func (s *capturingSkillRepoStub) CreateBlob(blob *models.SkillBlob) error { + if s.blobs == nil { + s.blobs = map[int]*models.SkillBlob{} + } + s.nextSkillID++ + blob.ID = s.nextSkillID + stored := *blob + s.blobs[blob.ID] = &stored + return nil +} + +func (s *capturingSkillRepoStub) CreateVersion(version *models.SkillVersion) error { + if s.versions == nil { + s.versions = map[int]*models.SkillVersion{} + } + s.nextSkillID++ + version.ID = s.nextSkillID + stored := *version + s.versions[version.ID] = &stored + return nil +} + +func (s *capturingSkillRepoStub) GetBlobByContentHash(hash string) (*models.SkillBlob, error) { + for _, blob := range s.blobs { + if blob != nil && blob.ContentHash == hash { + copy := *blob + return ©, nil + } + } + return nil, nil +} + +func (s *capturingSkillRepoStub) GetVersionBySkillAndBlob(skillID, blobID int) (*models.SkillVersion, error) { + for _, version := range s.versions { + if version != nil && version.SkillID == skillID && version.BlobID == blobID { + copy := *version + return ©, nil + } + } + return nil, nil +} + +func (s *capturingSkillRepoStub) UpsertInstanceSkill(item *models.InstanceSkill) error { + copy := *item + updated := false + for i, existing := range s.instanceSkills { + if existing.InstanceID == item.InstanceID && existing.SkillID == item.SkillID { + s.instanceSkills[i] = copy + updated = true + break + } + } + if !updated { + s.instanceSkills = append(s.instanceSkills, copy) + } + return nil +} + +func (s *capturingSkillRepoStub) MarkMissingInstanceSkills(instanceID int, activeSkillIDs []int, observedAt time.Time) error { + s.markMissingCalls++ + s.lastMarkMissingActive = append([]int(nil), activeSkillIDs...) + active := map[int]struct{}{} + for _, id := range activeSkillIDs { + active[id] = struct{}{} + } + for i := range s.instanceSkills { + item := &s.instanceSkills[i] + if item.InstanceID != instanceID { + continue + } + if _, ok := active[item.SkillID]; ok { + continue + } + if strings.EqualFold(item.Status, "removed") { + continue + } + item.Status = "missing" + item.RemovedAt = &observedAt + item.UpdatedAt = observedAt + } + return nil +} + +type importLiteMaterializer struct { + repo *skillRepoStub + storage *importTestObjectStorage + blobID int +} + +func (m importLiteMaterializer) materializeSkillPackageFromWorkspace(_ context.Context, instanceID int, workspaceDir, contentHash string, _ int) (*models.SkillBlob, error) { + objectKey := fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, workspaceDir, contentHash) + if m.storage.objects == nil { + m.storage.objects = map[string][]byte{} + } + m.storage.objects[objectKey] = []byte("fake-zip") + scanID := 99 + blob := m.repo.blobs[m.blobID] + blob.ObjectKey = objectKey + blob.ScanStatus = "completed" + blob.RiskLevel = skillRiskNone + blob.LastScanResultID = &scanID + m.repo.blobs[m.blobID] = blob + return blob, nil +} + +func (importLiteMaterializer) syncSkillRecordFromBlob(int, *models.SkillBlob) error { return nil } + +type importTestObjectStorage struct { + objects map[string][]byte +} + +func (s *importTestObjectStorage) PutObject(_ context.Context, objectKey string, body []byte, _ string) error { + if s.objects == nil { + s.objects = map[string][]byte{} + } + s.objects[objectKey] = body + return nil +} + +func (s *importTestObjectStorage) GetObject(_ context.Context, objectKey string) ([]byte, error) { + if body, ok := s.objects[objectKey]; ok { + return body, nil + } + return nil, fmt.Errorf("object not found: %s", objectKey) +} + +func TestDeleteSkillReleasesSkillKey(t *testing.T) { + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: {ID: 1, UserID: 1, SkillKey: "weather", Name: "Weather", Status: skillStatusActive, SourceType: skillSourceUploaded}, + }, + tagAssignments: map[int][]int{}, + } + svc := &skillService{repo: stub} + if err := svc.DeleteSkill(1, "user", 1); err != nil { + t.Fatalf("DeleteSkill() error = %v", err) + } + want := deletedSkillKey("weather", 1) + if stub.skills[1].SkillKey != want { + t.Fatalf("skill_key = %q, want %q", stub.skills[1].SkillKey, want) + } + active, err := stub.GetSkillByUserKey(1, "weather") + if err != nil { + t.Fatalf("GetSkillByUserKey() error = %v", err) + } + if active != nil { + t.Fatal("original skill_key should be released for re-import") + } +} + +func TestSyncAgentSkillsIncrementalDoesNotMarkMissing(t *testing.T) { + contentHash := "abc123def456789012345678901234" + versionID := 1 + stub := &capturingSkillRepoStub{ + skillRepoStub: skillRepoStub{ + skills: map[int]*models.Skill{ + 10: { + ID: 10, UserID: 1, SkillKey: "weather", Name: "weather", + SourceType: skillSourceDiscovered, Status: skillStatusActive, + Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + blobs: map[int]*models.SkillBlob{ + 1: {ID: 1, ContentHash: contentHash, ObjectKey: "discovered/weather.zip", ScanStatus: "completed"}, + }, + versions: map[int]*models.SkillVersion{ + 1: {ID: 1, SkillID: 10, BlobID: 1, VersionNo: 1}, + }, + instanceSkills: []models.InstanceSkill{ + {InstanceID: 1, SkillID: 10, Status: "active", SourceType: "discovered_in_instance"}, + }, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + + if err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{Mode: "incremental", Skills: nil}); err != nil { + t.Fatalf("incremental SyncAgentSkills() error = %v", err) + } + if stub.markMissingCalls != 0 { + t.Fatalf("markMissingCalls = %d, want 0 for incremental", stub.markMissingCalls) + } + if stub.instanceSkills[0].Status != "active" { + t.Fatalf("status = %q, want active after incremental empty report", stub.instanceSkills[0].Status) + } + + if err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{Mode: "full", Skills: nil}); err != nil { + t.Fatalf("full SyncAgentSkills() error = %v", err) + } + if stub.markMissingCalls != 1 { + t.Fatalf("markMissingCalls = %d, want 1 for full", stub.markMissingCalls) + } + if stub.instanceSkills[0].Status != "missing" { + t.Fatalf("status = %q, want missing after full empty report", stub.instanceSkills[0].Status) + } +} + +func TestSyncAgentSkillsReactivatesMissingButNotRemoved(t *testing.T) { + contentHash := "abc123def456789012345678901234" + versionID := 1 + removedAt := time.Now().UTC().Add(-time.Hour) + stub := &capturingSkillRepoStub{ + skillRepoStub: skillRepoStub{ + skills: map[int]*models.Skill{ + 10: { + ID: 10, UserID: 1, SkillKey: "weather", Name: "weather", + SourceType: skillSourceDiscovered, Status: skillStatusActive, + Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + 11: { + ID: 11, UserID: 1, SkillKey: "calendar", Name: "calendar", + SourceType: skillSourceDiscovered, Status: skillStatusActive, + Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + blobs: map[int]*models.SkillBlob{ + 1: {ID: 1, ContentHash: contentHash, ObjectKey: "discovered/weather.zip", ScanStatus: "completed"}, + 2: {ID: 2, ContentHash: "def456abc123789012345678901234", ObjectKey: "discovered/calendar.zip", ScanStatus: "completed"}, + }, + versions: map[int]*models.SkillVersion{ + 1: {ID: 1, SkillID: 10, BlobID: 1, VersionNo: 1}, + 2: {ID: 2, SkillID: 11, BlobID: 2, VersionNo: 1}, + }, + instanceSkills: []models.InstanceSkill{ + {InstanceID: 1, SkillID: 10, Status: "missing", SourceType: "discovered_in_instance", RemovedAt: &removedAt}, + {InstanceID: 1, SkillID: 11, Status: "removed", SourceType: "discovered_in_instance", RemovedAt: &removedAt}, + }, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + + err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{ + Mode: "full", + Skills: []AgentSkillRecord{ + {Identifier: "weather", ContentMD5: contentHash, Source: "discovered_in_instance"}, + {Identifier: "calendar", ContentMD5: "def456abc123789012345678901234", Source: "discovered_in_instance"}, + }, + }) + if err != nil { + t.Fatalf("SyncAgentSkills() error = %v", err) + } + + var missing, removed *models.InstanceSkill + for i := range stub.instanceSkills { + item := &stub.instanceSkills[i] + switch item.SkillID { + case 10: + missing = item + case 11: + removed = item + } + } + if missing == nil || missing.Status != "active" || missing.RemovedAt != nil { + t.Fatalf("missing skill revive = %#v, want active with nil RemovedAt", missing) + } + if removed == nil || removed.Status != "removed" { + t.Fatalf("removed skill = %#v, want status removed", removed) + } +} + +func TestDownloadSkillNilSafe(t *testing.T) { + missingVersionID := 999 + versionID := 7 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: {ID: 1, UserID: 1, Status: skillStatusActive, Visibility: skillVisibilityPrivate, CurrentVersionID: &missingVersionID}, + 2: {ID: 2, UserID: 1, Status: skillStatusActive, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID}, + }, + versions: map[int]*models.SkillVersion{ + 7: {ID: 7, SkillID: 2, BlobID: 99}, + }, + blobs: map[int]*models.SkillBlob{}, + } + svc := &skillService{repo: stub, storage: &importTestObjectStorage{objects: map[string][]byte{}}} + + if _, _, err := svc.DownloadSkill(1, "user", 1); err == nil { + t.Fatal("expected error when current version is missing") + } + if _, _, err := svc.DownloadSkill(1, "user", 2); err == nil { + t.Fatal("expected error when blob is missing") + } +} diff --git a/backend/internal/services/skill_import.go b/backend/internal/services/skill_import.go new file mode 100644 index 0000000..ac5e591 --- /dev/null +++ b/backend/internal/services/skill_import.go @@ -0,0 +1,439 @@ +package services + +import ( + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "strings" + "time" + + "clawreef/internal/models" +) + +const ( + skillImportConflictNone = "none" + skillImportConflictUnchanged = "unchanged" + skillImportConflictContentChanged = "content_changed" + + skillImportActionAuto = "auto" + skillImportActionNewVersion = "new_version" + skillImportActionSaveAsNew = "save_as_new" + skillImportActionSkip = "skip" + + skillImportResultCreated = "created" + skillImportResultVersioned = "versioned" + skillImportResultUnchanged = "unchanged" + skillImportResultSavedAsNew = "saved_as_new" +) + +type SkillImportPreviewItem struct { + DirectoryName string `json:"directory_name"` + SkillKey string `json:"skill_key"` + ContentHash string `json:"content_hash"` + ConflictType string `json:"conflict_type"` + ExistingSkillID *int `json:"existing_skill_id,omitempty"` + ExistingName *string `json:"existing_name,omitempty"` + CurrentVersionNo *int `json:"current_version_no,omitempty"` + SuggestedSkillKey *string `json:"suggested_skill_key,omitempty"` +} + +type SkillImportDecision struct { + DirectoryName string `json:"directory_name"` + Action string `json:"action"` + SkillKey *string `json:"skill_key,omitempty"` +} + +type SkillImportResultItem struct { + Skill SkillPayload `json:"skill"` + Action string `json:"action"` + PreviousVersionNo *int `json:"previous_version_no,omitempty"` + DirectoryName string `json:"directory_name"` +} + +type ImportDirectoryOptions struct { + Action string + OverrideSkillKey string +} + +func (s *skillService) PreviewHubImport(ctx context.Context, userID int, fileHeader *multipart.FileHeader) ([]SkillImportPreviewItem, error) { + _ = ctx + directories, _, err := readSkillArchiveDirectories(fileHeader) + if err != nil { + return nil, err + } + items := make([]SkillImportPreviewItem, 0, len(directories)) + for _, dir := range directories { + item, err := s.previewImportDirectory(userID, dir) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil +} + +func (s *skillService) ImportHubArchiveWithDecisions(ctx context.Context, userID int, fileHeader *multipart.FileHeader, decisions []SkillImportDecision) ([]SkillImportResultItem, error) { + directories, filename, err := readSkillArchiveDirectories(fileHeader) + if err != nil { + return nil, err + } + decisionMap := mapSkillImportDecisions(decisions) + hasDecisions := len(decisions) > 0 + + results := make([]SkillImportResultItem, 0, len(directories)) + for _, dir := range directories { + preview, err := s.previewImportDirectory(userID, dir) + if err != nil { + return nil, err + } + opts := resolveImportDecision(preview, decisionMap[dir.Name], hasDecisions) + + if preview.ConflictType == skillImportConflictUnchanged || opts.Action == skillImportActionSkip { + if preview.ConflictType == skillImportConflictUnchanged && preview.ExistingSkillID != nil { + payload, err := s.loadSkillPayloadByID(*preview.ExistingSkillID) + if err != nil { + return nil, err + } + results = append(results, SkillImportResultItem{ + Skill: *payload, + Action: skillImportResultUnchanged, + DirectoryName: dir.Name, + }) + } + continue + } + + result, err := s.importDirectoryWithOptions(ctx, userID, dir, filename, opts) + if err != nil { + return nil, err + } + result.DirectoryName = dir.Name + results = append(results, *result) + } + return results, nil +} + +func readSkillArchiveDirectories(fileHeader *multipart.FileHeader) ([]extractedSkillDirectory, string, error) { + if !strings.HasSuffix(strings.ToLower(strings.TrimSpace(fileHeader.Filename)), ".zip") { + return nil, "", fmt.Errorf("only .zip skill archives are supported") + } + file, err := fileHeader.Open() + if err != nil { + return nil, "", fmt.Errorf("failed to open uploaded archive: %w", err) + } + defer file.Close() + + raw, err := io.ReadAll(file) + if err != nil { + return nil, "", fmt.Errorf("failed to read uploaded archive: %w", err) + } + directories, err := extractSkillDirectories(fileHeader.Filename, raw) + if err != nil { + return nil, "", err + } + if len(directories) == 0 { + return nil, "", fmt.Errorf("no skill directories found in archive") + } + return directories, fileHeader.Filename, nil +} + +func mapSkillImportDecisions(decisions []SkillImportDecision) map[string]SkillImportDecision { + result := make(map[string]SkillImportDecision, len(decisions)) + for _, item := range decisions { + key := strings.TrimSpace(item.DirectoryName) + if key == "" { + continue + } + result[key] = item + } + return result +} + +func resolveImportDecision(preview SkillImportPreviewItem, decision SkillImportDecision, hasDecision bool) ImportDirectoryOptions { + if preview.ConflictType == skillImportConflictUnchanged { + return ImportDirectoryOptions{Action: skillImportActionSkip} + } + if !hasDecision { + return ImportDirectoryOptions{Action: skillImportActionAuto} + } + if strings.TrimSpace(decision.DirectoryName) == "" { + switch preview.ConflictType { + case skillImportConflictNone: + return ImportDirectoryOptions{Action: skillImportActionNewVersion} + case skillImportConflictContentChanged: + return ImportDirectoryOptions{Action: skillImportActionNewVersion} + default: + return ImportDirectoryOptions{Action: skillImportActionSkip} + } + } + switch strings.TrimSpace(decision.Action) { + case skillImportActionSaveAsNew: + key := preview.SkillKey + if preview.SuggestedSkillKey != nil && strings.TrimSpace(*preview.SuggestedSkillKey) != "" { + key = *preview.SuggestedSkillKey + } + if decision.SkillKey != nil && strings.TrimSpace(*decision.SkillKey) != "" { + key = strings.TrimSpace(*decision.SkillKey) + } + return ImportDirectoryOptions{Action: skillImportActionSaveAsNew, OverrideSkillKey: key} + case skillImportActionSkip: + return ImportDirectoryOptions{Action: skillImportActionSkip} + default: + return ImportDirectoryOptions{Action: skillImportActionNewVersion} + } +} + +func (s *skillService) previewImportDirectory(userID int, dir extractedSkillDirectory) (SkillImportPreviewItem, error) { + skillKey := sanitizeSkillKey(dir.Name) + if skillKey == "" { + return SkillImportPreviewItem{}, fmt.Errorf("skill directory name %q is invalid", dir.Name) + } + contentHash := hashDirectory(dir.Files) + item := SkillImportPreviewItem{ + DirectoryName: dir.Name, + SkillKey: skillKey, + ContentHash: contentHash, + ConflictType: skillImportConflictNone, + } + + skill, err := s.repo.GetSkillByUserKey(userID, skillKey) + if err != nil { + return SkillImportPreviewItem{}, err + } + if skill == nil { + return item, nil + } + + existingName := skill.Name + item.ExistingSkillID = &skill.ID + item.ExistingName = &existingName + + existingHash, versionNo, err := s.currentSkillContentHash(skill) + if err != nil { + return SkillImportPreviewItem{}, err + } + if versionNo != nil { + item.CurrentVersionNo = versionNo + } + if existingHash != "" && existingHash == contentHash { + item.ConflictType = skillImportConflictUnchanged + return item, nil + } + item.ConflictType = skillImportConflictContentChanged + suggested := s.nextUploadSkillKey(userID, skillKey) + item.SuggestedSkillKey = &suggested + return item, nil +} + +func (s *skillService) currentSkillContentHash(skill *models.Skill) (string, *int, error) { + if skill == nil || skill.CurrentVersionID == nil { + return "", nil, nil + } + version, err := s.repo.GetVersionByID(*skill.CurrentVersionID) + if err != nil { + return "", nil, err + } + if version == nil { + return "", nil, nil + } + versionNo := version.VersionNo + blob, err := s.repo.GetBlobByID(version.BlobID) + if err != nil { + return "", nil, err + } + if blob == nil { + return "", &versionNo, nil + } + return blob.ContentHash, &versionNo, nil +} + +func (s *skillService) loadSkillPayloadByID(skillID int) (*SkillPayload, error) { + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return nil, err + } + if skill == nil { + return nil, fmt.Errorf("skill not found") + } + payload, err := s.toSkillPayload(*skill) + if err != nil { + return nil, err + } + if err := s.enrichSkillPayload(payload, *skill, nil); err != nil { + return nil, err + } + return payload, nil +} + +func (s *skillService) nextUploadSkillKey(userID int, baseKey string) string { + candidate := strings.TrimSpace(baseKey) + if candidate == "" { + candidate = "skill" + } + for i := 2; i <= 99; i++ { + next := fmt.Sprintf("%s-%d", candidate, i) + existing, err := s.repo.GetSkillByUserKey(userID, next) + if err == nil && existing == nil { + return next + } + } + return fmt.Sprintf("%s-%d", candidate, time.Now().UTC().Unix()) +} + +func (s *skillService) importDirectory(ctx context.Context, userID int, dir extractedSkillDirectory, originalName string) (*SkillPayload, error) { + result, err := s.importDirectoryWithOptions(ctx, userID, dir, originalName, ImportDirectoryOptions{Action: skillImportActionAuto}) + if err != nil { + return nil, err + } + return &result.Skill, nil +} + +func (s *skillService) importDirectoryWithOptions(ctx context.Context, userID int, dir extractedSkillDirectory, originalName string, opts ImportDirectoryOptions) (*SkillImportResultItem, error) { + baseSkillKey := sanitizeSkillKey(dir.Name) + if baseSkillKey == "" { + return nil, fmt.Errorf("skill directory name %q is invalid", dir.Name) + } + + targetSkillKey := baseSkillKey + isSaveAsNew := opts.Action == skillImportActionSaveAsNew + if isSaveAsNew { + targetSkillKey = sanitizeSkillKey(opts.OverrideSkillKey) + if targetSkillKey == "" { + return nil, fmt.Errorf("invalid skill key for save_as_new") + } + } + + contentHash := hashDirectory(dir.Files) + archiveBytes, archiveHash, err := buildNormalizedZip(dir) + if err != nil { + return nil, err + } + + blob, err := s.repo.GetBlobByContentHash(contentHash) + if err != nil { + return nil, err + } + if blob == nil { + blob = &models.SkillBlob{ + ContentHash: contentHash, ArchiveHash: archiveHash, + ObjectKey: fmt.Sprintf("%d/%s/%s.zip", userID, targetSkillKey, contentHash), + FileName: fmt.Sprintf("%s.zip", targetSkillKey), + MediaType: "application/zip", SizeBytes: int64(len(archiveBytes)), + ScanStatus: "pending", RiskLevel: skillRiskUnknown, + } + if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil { + return nil, err + } + if err := s.repo.CreateBlob(blob); err != nil { + return nil, err + } + if err := s.recordScan(blob, &dir); err != nil { + return nil, err + } + } else { + if err := s.ensureBlobObject(ctx, blob, archiveBytes); err != nil { + return nil, err + } + if blob.LastScanResultID == nil || blob.ScanStatus != "completed" { + if err := s.recordScan(blob, &dir); err != nil { + return nil, err + } + } + } + + existingBefore, err := s.repo.GetSkillByUserKey(userID, targetSkillKey) + if err != nil { + return nil, err + } + var previousVersionNo *int + if existingBefore != nil { + _, versionNo, err := s.currentSkillContentHash(existingBefore) + if err != nil { + return nil, err + } + previousVersionNo = versionNo + } + + if isSaveAsNew && existingBefore != nil { + return nil, fmt.Errorf("skill key %q already exists", targetSkillKey) + } + + skill := existingBefore + created := false + if skill == nil { + description := fmt.Sprintf("Imported from %s", originalName) + skill = &models.Skill{ + UserID: userID, SkillKey: targetSkillKey, Name: dir.Name, Description: &description, + SourceType: skillSourceUploaded, Status: "active", Visibility: skillVisibilityPrivate, RiskLevel: blob.RiskLevel, + LastScannedAt: blob.LastScannedAt, LastScanResultID: blob.LastScanResultID, + } + if err := s.repo.CreateSkill(skill); err != nil { + return nil, err + } + created = true + } + + version, err := s.repo.GetVersionBySkillAndBlob(skill.ID, blob.ID) + if err != nil { + return nil, err + } + versionCreated := false + if version == nil { + latest, err := s.repo.GetLatestVersionBySkillID(skill.ID) + if err != nil { + return nil, err + } + versionNo := 1 + if latest != nil { + versionNo = latest.VersionNo + 1 + } + manifest, _ := json.Marshal(map[string]interface{}{"root_dir": dir.Name, "files": len(dir.Files)}) + manifestJSON := string(manifest) + version = &models.SkillVersion{ + SkillID: skill.ID, BlobID: blob.ID, VersionNo: versionNo, ManifestJSON: &manifestJSON, SourceType: skillSourceUploaded, + } + if err := s.repo.CreateVersion(version); err != nil { + return nil, err + } + versionCreated = true + } + + skill.CurrentVersionID = &version.ID + skill.RiskLevel = blob.RiskLevel + skill.LastScannedAt = blob.LastScannedAt + skill.LastScanResultID = blob.LastScanResultID + skill.UpdatedAt = time.Now().UTC() + if err := s.repo.UpdateSkill(skill); err != nil { + return nil, err + } + + payload, err := s.toSkillPayload(*skill) + if err != nil { + return nil, err + } + if err := s.enrichSkillPayload(payload, *skill, nil); err != nil { + return nil, err + } + + action := skillImportResultVersioned + if created || isSaveAsNew { + action = skillImportResultCreated + if isSaveAsNew { + action = skillImportResultSavedAsNew + } + } else if !versionCreated { + action = skillImportResultUnchanged + } + + result := &SkillImportResultItem{ + Skill: *payload, + Action: action, + DirectoryName: dir.Name, + } + if action == skillImportResultVersioned && previousVersionNo != nil { + result.PreviousVersionNo = previousVersionNo + } + return result, nil +} diff --git a/backend/internal/services/skill_import_test.go b/backend/internal/services/skill_import_test.go new file mode 100644 index 0000000..b469cc1 --- /dev/null +++ b/backend/internal/services/skill_import_test.go @@ -0,0 +1,75 @@ +package services + +import ( + "testing" + + "clawreef/internal/models" +) + +func TestPreviewImportDirectoryNone(t *testing.T) { + svc := &skillService{repo: &skillRepoStub{skills: map[int]*models.Skill{}}} + item, err := svc.previewImportDirectory(1, extractedSkillDirectory{ + Name: "weather", + Files: map[string][]byte{"SKILL.md": []byte("# weather")}, + }) + if err != nil { + t.Fatalf("previewImportDirectory() error = %v", err) + } + if item.ConflictType != skillImportConflictNone { + t.Fatalf("conflict_type = %q, want %q", item.ConflictType, skillImportConflictNone) + } +} + +func TestPreviewImportDirectoryUnchanged(t *testing.T) { + versionID := 10 + blobID := 20 + dir := extractedSkillDirectory{Name: "weather", Files: map[string][]byte{"SKILL.md": []byte("# weather")}} + contentHash := hashDirectory(dir.Files) + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "weather", Name: "Weather", Status: skillStatusActive, + SourceType: skillSourceUploaded, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, VersionNo: 2}}, + blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ContentHash: contentHash, ScanStatus: "completed"}}, + } + svc := &skillService{repo: stub} + item, err := svc.previewImportDirectory(1, dir) + if err != nil { + t.Fatalf("previewImportDirectory() error = %v", err) + } + if item.ConflictType != skillImportConflictUnchanged { + t.Fatalf("conflict_type = %q, want %q", item.ConflictType, skillImportConflictUnchanged) + } +} + +func TestPreviewImportDirectoryContentChanged(t *testing.T) { + versionID := 10 + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "weather", Name: "Weather", Status: skillStatusActive, + SourceType: skillSourceUploaded, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, VersionNo: 2}}, + blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ContentHash: "old-hash", ScanStatus: "completed"}}, + } + svc := &skillService{repo: stub} + item, err := svc.previewImportDirectory(1, extractedSkillDirectory{ + Name: "weather", + Files: map[string][]byte{"SKILL.md": []byte("# changed")}, + }) + if err != nil { + t.Fatalf("previewImportDirectory() error = %v", err) + } + if item.ConflictType != skillImportConflictContentChanged { + t.Fatalf("conflict_type = %q, want %q", item.ConflictType, skillImportConflictContentChanged) + } + if item.SuggestedSkillKey == nil || *item.SuggestedSkillKey != "weather-2" { + t.Fatalf("suggested skill key = %v, want weather-2", item.SuggestedSkillKey) + } +} diff --git a/backend/internal/services/skill_lite_package_collect.go b/backend/internal/services/skill_lite_package_collect.go new file mode 100644 index 0000000..fd59090 --- /dev/null +++ b/backend/internal/services/skill_lite_package_collect.go @@ -0,0 +1,242 @@ +package services + +import ( + "context" + "fmt" + "strings" + "time" + + "clawreef/internal/models" +) + +func loadLiteSkillDirectoryFromWorkspace(instance *models.Instance, workspaceDir string) (extractedSkillDirectory, string, error) { + workspaceDir = sanitizeWorkspaceRelativePath(strings.TrimSpace(workspaceDir)) + if workspaceDir == "" { + return extractedSkillDirectory{}, "", fmt.Errorf("workspace skill directory is required") + } + root := runtimeSkillInstallRoot(instance) + if root == "" { + return extractedSkillDirectory{}, "", fmt.Errorf("runtime skill workspace root is not configured") + } + skillRoot, err := joinRuntimeSkillPath(root, workspaceDir) + if err != nil { + return extractedSkillDirectory{}, "", fmt.Errorf("workspace skill directory is invalid: %s", workspaceDir) + } + files, err := collectLiteSkillDirectoryFiles(skillRoot) + if err != nil { + return extractedSkillDirectory{}, "", err + } + if len(files) == 0 { + return extractedSkillDirectory{}, "", fmt.Errorf("lite skill directory not found: %s", workspaceDir) + } + dir := extractedSkillDirectory{Name: workspaceDir, Files: files} + return dir, hashDirectory(files), nil +} + +func resolveLiteWorkspaceDir(instanceSkill *models.InstanceSkill, skill *models.Skill) string { + if instanceSkill != nil && instanceSkill.WorkspaceDir != nil && strings.TrimSpace(*instanceSkill.WorkspaceDir) != "" { + return sanitizeWorkspaceRelativePath(strings.TrimSpace(*instanceSkill.WorkspaceDir)) + } + if instanceSkill != nil { + if key := skillKeyForRemoval(instanceSkill); key != "" && !strings.HasPrefix(key, "skill-") { + return sanitizeWorkspaceRelativePath(key) + } + } + if skill != nil && strings.TrimSpace(skill.Name) != "" { + return sanitizeWorkspaceRelativePath(strings.TrimSpace(skill.Name)) + } + if skill != nil { + return sanitizeWorkspaceRelativePath(strings.TrimSpace(skill.SkillKey)) + } + return "" +} + +func (s *skillService) persistDiscoveredSkillPackage(ctx context.Context, instanceID int, dir extractedSkillDirectory, contentMD5 string, existingBlob *models.SkillBlob) (*models.SkillBlob, error) { + if s == nil || s.storage == nil { + return nil, fmt.Errorf("object storage is not configured") + } + contentMD5 = strings.TrimSpace(contentMD5) + if contentMD5 == "" { + return nil, fmt.Errorf("content hash is required") + } + + archiveBytes, archiveHash, err := buildNormalizedZip(dir) + if err != nil { + return nil, err + } + + blob := existingBlob + if blob == nil { + blob, err = s.repo.GetBlobByContentHash(contentMD5) + if err != nil { + return nil, err + } + } + if blob == nil { + blob = &models.SkillBlob{ + ContentHash: contentMD5, + ArchiveHash: archiveHash, + ObjectKey: fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, sanitizeSkillKey(dir.Name), contentMD5), + FileName: fmt.Sprintf("%s.zip", sanitizeSkillKey(dir.Name)), + MediaType: "application/zip", + SizeBytes: int64(len(archiveBytes)), + ScanStatus: "pending", + RiskLevel: skillRiskUnknown, + } + if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil { + return nil, err + } + if err := s.repo.CreateBlob(blob); err != nil { + return nil, err + } + } else if strings.TrimSpace(blob.ObjectKey) == "" { + blob.ObjectKey = fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, sanitizeSkillKey(dir.Name), contentMD5) + blob.FileName = fmt.Sprintf("%s.zip", sanitizeSkillKey(dir.Name)) + blob.MediaType = "application/zip" + blob.SizeBytes = int64(len(archiveBytes)) + blob.ArchiveHash = archiveHash + if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil { + return nil, err + } + if err := s.repo.UpdateBlob(blob); err != nil { + return nil, err + } + } + + if blob.LastScanResultID == nil || !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") { + if err := s.recordScan(blob, &dir); err != nil { + blob.ScanStatus = "failed" + blob.UpdatedAt = timeNowUTC() + _ = s.repo.UpdateBlob(blob) + } + } + updated, err := s.repo.GetBlobByID(blob.ID) + if err != nil { + return nil, err + } + if updated != nil { + blob = updated + } + return blob, nil +} + +func (s *skillService) materializeSkillPackageFromWorkspace(ctx context.Context, instanceID int, workspaceDir, expectedMD5 string, targetBlobID int) (*models.SkillBlob, error) { + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, err + } + if instance == nil { + return nil, fmt.Errorf("instance not found") + } + if !isLiteRuntimeInstance(instance) && !SupportsServerWorkspaceSkillScan(instance) { + return nil, fmt.Errorf("instance does not support workspace skill materialization") + } + + dir, contentMD5, err := loadLiteSkillDirectoryFromWorkspace(instance, workspaceDir) + if err != nil { + return nil, err + } + + var existingBlob *models.SkillBlob + if targetBlobID > 0 { + existingBlob, err = s.repo.GetBlobByID(targetBlobID) + if err != nil { + return nil, err + } + } + if existingBlob == nil { + existingBlob, err = s.repo.GetBlobByContentHash(contentMD5) + if err != nil { + return nil, err + } + } + if existingBlob != nil && !strings.EqualFold(strings.TrimSpace(existingBlob.ContentHash), contentMD5) { + existingBlob.ContentHash = contentMD5 + existingBlob.ArchiveHash = contentMD5 + } + if existingBlob != nil && strings.TrimSpace(existingBlob.ObjectKey) != "" && strings.EqualFold(strings.TrimSpace(existingBlob.ScanStatus), "completed") { + return existingBlob, nil + } + return s.persistDiscoveredSkillPackage(ctx, instanceID, dir, contentMD5, existingBlob) +} + +func (s *skillService) reconcileLiteDiscoveredBlob(skill *models.Skill, contentHash string) (*models.SkillBlob, *models.SkillVersion, error) { + if s == nil || s.repo == nil || skill == nil || skill.CurrentVersionID == nil { + return nil, nil, nil + } + contentHash = strings.TrimSpace(contentHash) + if contentHash == "" { + return nil, nil, nil + } + version, err := s.repo.GetVersionByID(*skill.CurrentVersionID) + if err != nil { + return nil, nil, err + } + if version == nil { + return nil, nil, nil + } + blob, err := s.repo.GetBlobByID(version.BlobID) + if err != nil { + return nil, version, err + } + if blob == nil { + return nil, version, nil + } + if !strings.EqualFold(strings.TrimSpace(blob.ContentHash), contentHash) { + blob.ContentHash = contentHash + blob.ArchiveHash = contentHash + if err := s.repo.UpdateBlob(blob); err != nil { + return nil, version, err + } + } + return blob, version, nil +} + +func liteInventoryUsesWorkspaceHash(instance *models.Instance) bool { + return isLiteRuntimeInstance(instance) || SupportsServerWorkspaceSkillScan(instance) +} + +func workspaceContentHashForRecord(instance *models.Instance, record AgentSkillRecord) string { + if !runtimeInventoryUsesWorkspaceHash(instance) { + return strings.TrimSpace(record.ContentMD5) + } + workspaceDir := sanitizeWorkspaceRelativePath(strings.TrimSpace(record.Identifier)) + if workspaceDir == "" { + return strings.TrimSpace(record.ContentMD5) + } + _, computed, err := loadLiteSkillDirectoryFromWorkspace(instance, workspaceDir) + if err != nil || strings.TrimSpace(computed) == "" { + return strings.TrimSpace(record.ContentMD5) + } + return computed +} + +func workspaceContentHashForLiteRecord(instance *models.Instance, record AgentSkillRecord) string { + return workspaceContentHashForRecord(instance, record) +} + +func runtimeInventoryUsesWorkspaceHash(instance *models.Instance) bool { + return liteInventoryUsesWorkspaceHash(instance) +} + +func (s *skillService) syncSkillRecordFromBlob(skillID int, blob *models.SkillBlob) error { + if blob == nil { + return nil + } + skill, err := s.repo.GetSkillByID(skillID) + if err != nil { + return err + } + if skill == nil { + return nil + } + skill.RiskLevel = blob.RiskLevel + skill.LastScannedAt = blob.LastScannedAt + skill.LastScanResultID = blob.LastScanResultID + skill.UpdatedAt = timeNowUTC() + return s.repo.UpdateSkill(skill) +} + +func timeNowUTC() time.Time { + return time.Now().UTC() +} diff --git a/backend/internal/services/skill_lite_package_collect_test.go b/backend/internal/services/skill_lite_package_collect_test.go new file mode 100644 index 0000000..d69068e --- /dev/null +++ b/backend/internal/services/skill_lite_package_collect_test.go @@ -0,0 +1,136 @@ +package services + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "clawreef/internal/models" +) + +func TestMaterializeSelfHealsStaleBlobContentHash(t *testing.T) { + root := t.TempDir() + workspace := filepath.Join(root, "hermes", "user-1", "instance-1") + skillRoot := filepath.Join(workspace, "home", ".hermes", "skills", "demo") + if err := os.MkdirAll(skillRoot, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# demo\n"), 0o640); err != nil { + t.Fatal(err) + } + + instance := &models.Instance{ + ID: 1, + UserID: 1, + Type: RuntimeTypeHermes, + InstanceMode: InstanceModeLite, + RuntimeType: RuntimeBackendGateway, + WorkspacePath: strPtr(workspace), + } + _, contentHash, err := loadLiteSkillDirectoryFromWorkspace(instance, "demo") + if err != nil { + t.Fatalf("loadLiteSkillDirectoryFromWorkspace() error = %v", err) + } + staleHash := "deadbeefdeadbeefdeadbeefdeadbeef" + + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: {ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, CurrentVersionID: intPtr(10)}, + }, + versions: map[int]*models.SkillVersion{10: {ID: 10, SkillID: 1, BlobID: blobID}}, + blobs: map[int]*models.SkillBlob{ + blobID: {ID: blobID, ContentHash: staleHash, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: instance}} + storage := &importTestObjectStorage{objects: map[string][]byte{}} + svc := &skillService{repo: stub, instanceRepo: instRepo, storage: storage, scanner: testSkillScanner{}} + + blob, err := svc.materializeSkillPackageFromWorkspace(context.Background(), 1, "demo", staleHash, blobID) + if err != nil { + t.Fatalf("materializeSkillPackageFromWorkspace() error = %v", err) + } + if blob == nil || strings.TrimSpace(blob.ObjectKey) == "" { + t.Fatalf("expected materialized blob, got %#v", blob) + } + if !strings.EqualFold(strings.TrimSpace(stub.blobs[blobID].ContentHash), contentHash) { + t.Fatalf("blob content_hash = %q, want %q", stub.blobs[blobID].ContentHash, contentHash) + } +} + +func TestWorkspaceContentHashForLiteRecordOverridesAgentMD5(t *testing.T) { + root := t.TempDir() + workspace := filepath.Join(root, "hermes", "user-1", "instance-1") + skillRoot := filepath.Join(workspace, "home", ".hermes", "skills", "yuanbao") + if err := os.MkdirAll(skillRoot, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# yuanbao\n"), 0o640); err != nil { + t.Fatal(err) + } + instance := &models.Instance{ + Type: RuntimeTypeHermes, InstanceMode: InstanceModeLite, + RuntimeType: RuntimeBackendGateway, WorkspacePath: strPtr(workspace), + } + got := workspaceContentHashForLiteRecord(instance, AgentSkillRecord{ + Identifier: "yuanbao", ContentMD5: "deadbeefdeadbeefdeadbeefdeadbeef", + }) + _, want, err := loadLiteSkillDirectoryFromWorkspace(instance, "yuanbao") + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("workspaceContentHashForLiteRecord() = %q, want %q", got, want) + } +} + +func TestResolveLiteWorkspaceDir(t *testing.T) { + workspace := "yuanbao" + instanceSkill := &models.InstanceSkill{ + WorkspaceDir: &workspace, + InstallPath: strPtr("home/.hermes/skills/yuanbao"), + } + skill := &models.Skill{SkillKey: "yuanbao-deadbeef", Name: "yuanbao"} + if got := resolveLiteWorkspaceDir(instanceSkill, skill); got != "yuanbao" { + t.Fatalf("resolveLiteWorkspaceDir() = %q, want yuanbao", got) + } +} + +func TestLoadLiteSkillDirectoryFromWorkspace(t *testing.T) { + root := t.TempDir() + workspace := filepath.Join(root, "hermes", "user-1", "instance-1") + skillRoot := filepath.Join(workspace, "home", ".hermes", "skills", "weather") + if err := os.MkdirAll(filepath.Join(skillRoot, "src"), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# weather\n"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillRoot, "src", "main.py"), []byte("print('ok')\n"), 0o640); err != nil { + t.Fatal(err) + } + + instance := &models.Instance{ + Type: RuntimeTypeHermes, + InstanceMode: InstanceModeLite, + RuntimeType: RuntimeBackendGateway, + WorkspacePath: strPtr(workspace), + } + dir, md5, err := loadLiteSkillDirectoryFromWorkspace(instance, "weather") + if err != nil { + t.Fatalf("loadLiteSkillDirectoryFromWorkspace() error = %v", err) + } + if dir.Name != "weather" || len(dir.Files) != 2 { + t.Fatalf("unexpected dir: %#v", dir) + } + if md5 == "" { + t.Fatal("expected non-empty md5") + } +} + +func strPtr(value string) *string { + return &value +} diff --git a/backend/internal/services/skill_package_collect_test.go b/backend/internal/services/skill_package_collect_test.go new file mode 100644 index 0000000..dd31aa4 --- /dev/null +++ b/backend/internal/services/skill_package_collect_test.go @@ -0,0 +1,68 @@ +package services + +import ( + "testing" + "time" + + "clawreef/internal/models" +) + +type stubCommandRepo struct { + failed *models.InstanceCommand +} + +func (s *stubCommandRepo) Create(*models.InstanceCommand) error { panic("not used") } +func (s *stubCommandRepo) Update(*models.InstanceCommand) error { panic("not used") } +func (s *stubCommandRepo) GetByID(int) (*models.InstanceCommand, error) { panic("not used") } +func (s *stubCommandRepo) GetByInstanceIdempotencyKey(int, string) (*models.InstanceCommand, error) { + panic("not used") +} +func (s *stubCommandRepo) GetNextPendingByInstance(int) (*models.InstanceCommand, error) { + panic("not used") +} +func (s *stubCommandRepo) ListByInstanceID(int, int) ([]models.InstanceCommand, error) { + panic("not used") +} +func (s *stubCommandRepo) FindLatestFailedCollectSkillPackage(string) (*models.InstanceCommand, error) { + return s.failed, nil +} + +func TestPublishBlockedReasonCollectFailed(t *testing.T) { + errMsg := `unexpected status 500: {"error":"skill package md5 mismatch: expected abc got def","success":false}` + svc := &skillService{ + commandRepo: &stubCommandRepo{ + failed: &models.InstanceCommand{ + CommandType: "collect_skill_package", + Status: "failed", + ErrorMessage: &errMsg, + FinishedAt: ptrTime(time.Now()), + }, + }, + } + skill := &models.Skill{ID: 2, Status: skillStatusActive, SourceType: skillSourceDiscovered} + blob := &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""} + + reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, false) + if reason == nil || *reason != "skill_package_collect_failed" { + t.Fatalf("expected skill_package_collect_failed, got %v", reason) + } + collectErr := svc.resolvePackageCollectError(skill.ID, blob, nil, false) + if collectErr == nil || *collectErr != errMsg { + t.Fatalf("expected package collect error summary, got %v", collectErr) + } +} + +func TestPublishBlockedReasonScanFailed(t *testing.T) { + svc := &skillService{} + skill := &models.Skill{ID: 1, Status: skillStatusActive, SourceType: skillSourceUploaded} + blob := &models.SkillBlob{ScanStatus: "failed", RiskLevel: skillRiskUnknown, ObjectKey: "discovered/1/demo.zip"} + + reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, false) + if reason == nil || *reason != "skill_scan_failed" { + t.Fatalf("expected skill_scan_failed, got %v", reason) + } +} + +func ptrTime(value time.Time) *time.Time { + return &value +} diff --git a/backend/internal/services/skill_package_materialize_service.go b/backend/internal/services/skill_package_materialize_service.go new file mode 100644 index 0000000..b6cf559 --- /dev/null +++ b/backend/internal/services/skill_package_materialize_service.go @@ -0,0 +1,298 @@ +package services + +import ( + "context" + "fmt" + "strings" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +const ( + MaterializeJobStatusPending = "pending" + MaterializeJobStatusRunning = "running" + MaterializeJobStatusSucceeded = "succeeded" + MaterializeJobStatusFailed = "failed" + MaterializeJobStatusCancelled = "cancelled" + MaterializeTriggerSync = "sync" + MaterializeTriggerRetry = "retry" + MaterializeTriggerImport = "import" + MaterializeTriggerPublish = "publish" + MaterializeTriggerBackfill = "backfill" +) + +type EnqueueMaterializeRequest struct { + InstanceID int + SkillID int + BlobID int + WorkspaceDir string + ContentHash string + TriggerSource string + IdempotencyKey string +} + +type skillBlobReader interface { + GetBlobByID(id int) (*models.SkillBlob, error) +} + +type SkillPackageMaterializeService struct { + jobRepo repository.SkillPackageMaterializeJobRepository + blobRepo skillBlobReader + worker skillPackageMaterializer +} + +type skillPackageMaterializer interface { + materializeSkillPackageFromWorkspace(ctx context.Context, instanceID int, workspaceDir, contentHash string, targetBlobID int) (*models.SkillBlob, error) + syncSkillRecordFromBlob(skillID int, blob *models.SkillBlob) error +} + +func SkillServiceAsMaterializer(service SkillService) skillPackageMaterializer { + if impl, ok := service.(*skillService); ok { + return impl + } + return nil +} + +func NewSkillPackageMaterializeService(jobRepo repository.SkillPackageMaterializeJobRepository, blobRepo skillBlobReader, worker skillPackageMaterializer) *SkillPackageMaterializeService { + return &SkillPackageMaterializeService{jobRepo: jobRepo, blobRepo: blobRepo, worker: worker} +} + +func (m *SkillPackageMaterializeService) Enqueue(ctx context.Context, req EnqueueMaterializeRequest) (*models.SkillPackageMaterializeJob, error) { + if m == nil || m.jobRepo == nil { + return nil, fmt.Errorf("skill package materialize service is not configured") + } + workspaceDir := sanitizeWorkspaceRelativePath(strings.TrimSpace(req.WorkspaceDir)) + contentHash := strings.TrimSpace(req.ContentHash) + if req.InstanceID <= 0 || req.SkillID <= 0 || req.BlobID <= 0 || workspaceDir == "" || contentHash == "" { + return nil, fmt.Errorf("invalid materialize enqueue request") + } + idempotencyKey := strings.TrimSpace(req.IdempotencyKey) + if idempotencyKey == "" { + idempotencyKey = fmt.Sprintf("materialize-%d-%s", req.InstanceID, contentHash) + } + if existing, err := m.jobRepo.GetByIdempotencyKey(idempotencyKey); err != nil { + return nil, err + } else if existing != nil { + if strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusSucceeded) { + return existing, nil + } + if strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusFailed) { + if err := m.jobRepo.RequeueExisting(existing.ID, req.BlobID, contentHash, workspaceDir); err != nil { + return nil, err + } + return m.jobRepo.GetByID(existing.ID) + } + if strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusPending) || + strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusRunning) { + return existing, nil + } + } + if m.blobRepo != nil { + blob, err := m.blobRepo.GetBlobByID(req.BlobID) + if err != nil { + return nil, err + } + if blob != nil && strings.TrimSpace(blob.ObjectKey) != "" { + if existing, err := m.jobRepo.GetByIdempotencyKey(idempotencyKey); err != nil { + return nil, err + } else if existing != nil { + if !strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusSucceeded) { + if err := m.jobRepo.MarkSucceeded(existing.ID); err != nil { + return nil, err + } + existing, err = m.jobRepo.GetByID(existing.ID) + if err != nil { + return nil, err + } + } + return existing, nil + } + job := &models.SkillPackageMaterializeJob{ + InstanceID: req.InstanceID, + SkillID: req.SkillID, + BlobID: req.BlobID, + WorkspaceDir: workspaceDir, + ContentHash: contentHash, + Status: MaterializeJobStatusSucceeded, + MaxAttempts: 5, + IdempotencyKey: idempotencyKey, + TriggerSource: strings.TrimSpace(req.TriggerSource), + } + if job.TriggerSource == "" { + job.TriggerSource = MaterializeTriggerSync + } + if err := m.jobRepo.Create(job); err != nil { + return nil, err + } + if !strings.EqualFold(strings.TrimSpace(job.Status), MaterializeJobStatusSucceeded) { + if err := m.jobRepo.MarkSucceeded(job.ID); err != nil { + return nil, err + } + job, err = m.jobRepo.GetByID(job.ID) + if err != nil { + return nil, err + } + } + return job, nil + } + } + trigger := strings.TrimSpace(req.TriggerSource) + if trigger == "" { + trigger = MaterializeTriggerSync + } + + job := &models.SkillPackageMaterializeJob{ + InstanceID: req.InstanceID, + SkillID: req.SkillID, + BlobID: req.BlobID, + WorkspaceDir: workspaceDir, + ContentHash: contentHash, + Status: MaterializeJobStatusPending, + MaxAttempts: 5, + IdempotencyKey: idempotencyKey, + TriggerSource: trigger, + } + if err := m.jobRepo.Create(job); err != nil { + return nil, err + } + return job, nil +} + +func (m *SkillPackageMaterializeService) ReleaseToPending(id int) error { + if m == nil || m.jobRepo == nil { + return fmt.Errorf("skill package materialize service is not configured") + } + return m.jobRepo.ReleaseToPending(id) +} + +func (m *SkillPackageMaterializeService) ClaimNextPending(ctx context.Context, limit int) ([]models.SkillPackageMaterializeJob, error) { + if m == nil || m.jobRepo == nil { + return nil, fmt.Errorf("skill package materialize service is not configured") + } + return m.jobRepo.ClaimNextPending(ctx, limit) +} + +func (m *SkillPackageMaterializeService) ProcessJob(ctx context.Context, jobID int) error { + if m == nil || m.jobRepo == nil || m.worker == nil { + return fmt.Errorf("skill package materialize service is not configured") + } + job, err := m.jobRepo.GetByID(jobID) + if err != nil { + return err + } + if job == nil { + return fmt.Errorf("materialize job not found") + } + if strings.EqualFold(strings.TrimSpace(job.Status), MaterializeJobStatusPending) { + if err := m.jobRepo.MarkRunning(jobID); err != nil { + return err + } + job, err = m.jobRepo.GetByID(jobID) + if err != nil { + return err + } + if job == nil { + return fmt.Errorf("materialize job not found") + } + } + + blob, err := m.worker.materializeSkillPackageFromWorkspace(ctx, job.InstanceID, job.WorkspaceDir, job.ContentHash, job.BlobID) + if err != nil { + retryable := !strings.Contains(strings.ToLower(err.Error()), "md5 mismatch") + if markErr := m.jobRepo.MarkFailed(job.ID, err.Error(), retryable); markErr != nil { + return markErr + } + return err + } + if err := m.worker.syncSkillRecordFromBlob(job.SkillID, blob); err != nil { + if markErr := m.jobRepo.MarkFailed(job.ID, err.Error(), true); markErr != nil { + return markErr + } + return err + } + return m.jobRepo.MarkSucceeded(job.ID) +} + +func (m *SkillPackageMaterializeService) RetryJob(skillID int) error { + if m == nil || m.jobRepo == nil { + return fmt.Errorf("skill package materialize service is not configured") + } + return m.jobRepo.ResetForRetry(skillID) +} + +func (m *SkillPackageMaterializeService) BackfillOnce(ctx context.Context, limit int) (int, error) { + if m == nil || m.jobRepo == nil { + return 0, fmt.Errorf("skill package materialize service is not configured") + } + candidates, err := m.jobRepo.ListBackfillCandidates(limit) + if err != nil { + return 0, err + } + enqueued := 0 + for _, candidate := range candidates { + _, err := m.Enqueue(ctx, EnqueueMaterializeRequest{ + InstanceID: candidate.InstanceID, + SkillID: candidate.SkillID, + BlobID: candidate.BlobID, + WorkspaceDir: candidate.WorkspaceDir, + ContentHash: candidate.ContentHash, + TriggerSource: MaterializeTriggerBackfill, + IdempotencyKey: fmt.Sprintf("materialize-%d-%s", candidate.InstanceID, candidate.ContentHash), + }) + if err != nil { + return enqueued, err + } + enqueued++ + } + return enqueued, nil +} + +func (m *SkillPackageMaterializeService) FindLatestBySkillID(skillID int) (*models.SkillPackageMaterializeJob, error) { + if m == nil || m.jobRepo == nil { + return nil, nil + } + return m.jobRepo.FindLatestBySkillID(skillID) +} + +func (m *SkillPackageMaterializeService) GetObservedStatus(skillID int, blob *models.SkillBlob) (*string, *string) { + if m == nil || m.jobRepo == nil { + return nil, nil + } + if blob != nil && strings.TrimSpace(blob.ObjectKey) != "" { + return nil, nil + } + job, err := m.jobRepo.FindLatestBySkillID(skillID) + if err != nil || job == nil { + return nil, nil + } + status := strings.TrimSpace(job.Status) + if status == "" { + return nil, nil + } + var errSummary *string + if job.LastError != nil && strings.TrimSpace(*job.LastError) != "" { + summary := truncateCollectError(*job.LastError, 512) + if summary != "" { + errSummary = &summary + } + } + return &status, errSummary +} + +func materializeBlockedReason(job *models.SkillPackageMaterializeJob) *string { + if job == nil { + return nil + } + reason := func(value string) *string { return &value } + switch strings.TrimSpace(job.Status) { + case MaterializeJobStatusRunning: + return reason("skill_package_materializing") + case MaterializeJobStatusPending: + return reason("skill_package_materializing") + case MaterializeJobStatusFailed: + return reason("skill_package_materialize_failed") + default: + return nil + } +} diff --git a/backend/internal/services/skill_package_materialize_service_test.go b/backend/internal/services/skill_package_materialize_service_test.go new file mode 100644 index 0000000..04891a6 --- /dev/null +++ b/backend/internal/services/skill_package_materialize_service_test.go @@ -0,0 +1,445 @@ +package services + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +func TestMaterializeBlockedReason(t *testing.T) { + running := "skill_package_materializing" + if got := materializeBlockedReason(&models.SkillPackageMaterializeJob{Status: MaterializeJobStatusRunning}); got == nil || *got != running { + t.Fatalf("running reason = %v, want %q", got, running) + } + failed := "skill_package_materialize_failed" + if got := materializeBlockedReason(&models.SkillPackageMaterializeJob{Status: MaterializeJobStatusFailed}); got == nil || *got != failed { + t.Fatalf("failed reason = %v, want %q", got, failed) + } +} + +func TestPublishBlockedReasonUsesMaterializeJob(t *testing.T) { + svc := &skillService{ + materializeService: NewSkillPackageMaterializeService( + &materializeJobRepoStub{latest: &models.SkillPackageMaterializeJob{Status: MaterializeJobStatusPending}}, + nil, + nil, + ), + } + skill := &models.Skill{ID: 1, Status: skillStatusActive, SourceType: skillSourceDiscovered} + blob := &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""} + + reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, false) + if reason == nil || *reason != "skill_package_materializing" { + t.Fatalf("expected skill_package_materializing, got %v", reason) + } +} + +func TestPublishBlockedReasonSkipsCollectWhenMaterializeJobSucceeded(t *testing.T) { + svc := &skillService{ + materializeService: NewSkillPackageMaterializeService( + &materializeJobRepoStub{latest: &models.SkillPackageMaterializeJob{Status: MaterializeJobStatusSucceeded}}, + nil, + nil, + ), + commandRepo: &stubCommandRepo{ + failed: &models.InstanceCommand{ + CommandType: "collect_skill_package", + Status: "failed", + ErrorMessage: strPtr("agent failed"), + }, + }, + } + skill := &models.Skill{ID: 1, Status: skillStatusActive, SourceType: skillSourceDiscovered} + blob := &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""} + + reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, false) + if reason == nil || *reason != "skill_package_pending" { + t.Fatalf("expected skill_package_pending, got %v", reason) + } +} + +func TestPublishBlockedReasonLiteSkipsAgentCollectFailed(t *testing.T) { + svc := &skillService{ + commandRepo: &stubCommandRepo{ + failed: &models.InstanceCommand{ + CommandType: "collect_skill_package", + Status: "failed", + ErrorMessage: strPtr("agent failed"), + }, + }, + } + skill := &models.Skill{ID: 1, Status: skillStatusActive, SourceType: skillSourceDiscovered} + blob := &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""} + + reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, true) + if reason == nil || *reason != "skill_package_pending" { + t.Fatalf("expected skill_package_pending, got %v", reason) + } + collectErr := svc.resolvePackageCollectError(skill.ID, blob, nil, true) + if collectErr != nil { + t.Fatalf("expected nil collect error for lite, got %v", collectErr) + } +} + +func TestListMyHubSkillsLiteAutoResolvesInstanceContext(t *testing.T) { + versionID := 10 + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "yuanbao", Name: "yuanbao", Status: skillStatusActive, + SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID, + }, + }, + versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}}, + blobs: map[int]*models.SkillBlob{ + blobID: {ID: blobID, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}, + }, + instanceSkills: []models.InstanceSkill{{ + InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance", + }}, + tagAssignments: map[int][]int{}, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{ + 1: {ID: 1, UserID: 1, InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway}, + }} + svc := &skillService{ + repo: stub, + instanceRepo: instRepo, + commandRepo: &stubCommandRepo{ + failed: &models.InstanceCommand{ + CommandType: "collect_skill_package", + Status: "failed", + ErrorMessage: strPtr("agent failed"), + }, + }, + } + items, err := svc.ListMyHubSkills(1) + if err != nil { + t.Fatalf("ListMyHubSkills() error = %v", err) + } + if len(items) != 1 { + t.Fatalf("items = %d, want 1", len(items)) + } + if items[0].PublishBlockedReason == nil || *items[0].PublishBlockedReason != "skill_package_pending" { + t.Fatalf("PublishBlockedReason = %v, want skill_package_pending", items[0].PublishBlockedReason) + } + if items[0].PackageCollectError != nil { + t.Fatalf("PackageCollectError = %v, want nil", items[0].PackageCollectError) + } +} + +func TestSyncAgentSkillsLiteSkipsAgentEnqueue(t *testing.T) { + stub := &capturingSkillRepoStub{ + skillRepoStub: skillRepoStub{ + skills: map[int]*models.Skill{}, + blobs: map[int]*models.SkillBlob{}, + versions: map[int]*models.SkillVersion{}, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{ + 1: {ID: 1, UserID: 1, InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway}, + }} + cmdSvc := &capturingInstanceCommandService{} + matSvc := NewSkillPackageMaterializeService(&materializeJobRepoStub{}, nil, nil) + svc := &skillService{ + repo: stub, + instanceRepo: instRepo, + commandService: cmdSvc, + materializeService: matSvc, + } + err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{ + Skills: []AgentSkillRecord{{ + Identifier: "yuanbao", + ContentMD5: "abc123def456789012345678901234", + Source: "discovered_in_instance", + InstallPath: "home/.hermes/skills/yuanbao", + }}, + }) + if err != nil { + t.Fatalf("SyncAgentSkills() error = %v", err) + } + for _, req := range cmdSvc.created { + if req.CommandType == InstanceCommandTypeCollectSkillPackage { + t.Fatalf("unexpected collect_skill_package command: %#v", req) + } + } +} + +func TestEnqueueSkipsWhenObjectKeyPresent(t *testing.T) { + objectKey := "discovered/1/demo/abc.zip" + service := NewSkillPackageMaterializeService( + &materializeJobRepoStub{}, + &materializeBlobRepoStub{ + blobs: map[int]*models.SkillBlob{ + 20: {ID: 20, ObjectKey: objectKey, ContentHash: "abc"}, + }, + }, + nil, + ) + job, err := service.Enqueue(context.Background(), EnqueueMaterializeRequest{ + InstanceID: 1, + SkillID: 1, + BlobID: 20, + WorkspaceDir: "demo", + ContentHash: "abc", + TriggerSource: MaterializeTriggerSync, + IdempotencyKey: "materialize-1-abc", + }) + if err != nil { + t.Fatalf("Enqueue() error = %v", err) + } + if job == nil || job.Status != MaterializeJobStatusSucceeded { + t.Fatalf("expected succeeded job, got %#v", job) + } +} + +func TestEnqueueRequeuesFailedMaterializeJob(t *testing.T) { + failed := "skill package md5 mismatch" + job := &models.SkillPackageMaterializeJob{ + ID: 7, InstanceID: 1, SkillID: 1, BlobID: 20, WorkspaceDir: "demo", + ContentHash: "stale", Status: MaterializeJobStatusFailed, LastError: &failed, + IdempotencyKey: "materialize-1-goodhash", + } + repo := &materializeJobRepoStub{ + latest: job, + byKey: map[string]*models.SkillPackageMaterializeJob{"materialize-1-goodhash": job}, + } + service := NewSkillPackageMaterializeService(repo, nil, nil) + updated, err := service.Enqueue(context.Background(), EnqueueMaterializeRequest{ + InstanceID: 1, + SkillID: 1, + BlobID: 20, + WorkspaceDir: "demo", + ContentHash: "goodhash", + IdempotencyKey: "materialize-1-goodhash", + }) + if err != nil { + t.Fatalf("Enqueue() error = %v", err) + } + if updated == nil || updated.Status != MaterializeJobStatusPending { + t.Fatalf("expected pending job, got %#v", updated) + } + if updated.ContentHash != "goodhash" { + t.Fatalf("content_hash = %q, want goodhash", updated.ContentHash) + } +} + +func TestNewSkillPackageMaterializeWorkerDefaults(t *testing.T) { + worker := NewSkillPackageMaterializeWorker(nil, 0, 0, 0, 0, true) + if worker.perInstanceLimit != 2 { + t.Fatalf("perInstanceLimit = %d, want 2", worker.perInstanceLimit) + } + if worker.concurrency != 5 { + t.Fatalf("concurrency = %d, want 5", worker.concurrency) + } +} + +type capturingInstanceCommandService struct { + created []CreateInstanceCommandRequest +} + +func (c *capturingInstanceCommandService) Create(_ int, _ *int, req CreateInstanceCommandRequest) (*InstanceCommandPayload, error) { + c.created = append(c.created, req) + return &InstanceCommandPayload{CommandType: req.CommandType, Status: "pending"}, nil +} +func (c *capturingInstanceCommandService) GetNextForAgent(*AgentSession) (*AgentCommandEnvelope, error) { + return nil, nil +} +func (c *capturingInstanceCommandService) MarkStarted(*AgentSession, int, *time.Time) error { return nil } +func (c *capturingInstanceCommandService) MarkFinished(*AgentSession, int, AgentCommandFinishRequest) error { + return nil +} +func (c *capturingInstanceCommandService) ListByInstanceID(int, int) ([]InstanceCommandPayload, error) { + return nil, nil +} + +type materializeJobRepoStub struct { + latest *models.SkillPackageMaterializeJob + created []*models.SkillPackageMaterializeJob + byKey map[string]*models.SkillPackageMaterializeJob +} + +func (s *materializeJobRepoStub) Create(job *models.SkillPackageMaterializeJob) error { + s.created = append(s.created, job) + if job.ID == 0 { + job.ID = len(s.created) + } + return nil +} +func (s *materializeJobRepoStub) GetByID(id int) (*models.SkillPackageMaterializeJob, error) { + for _, job := range s.created { + if job.ID == id { + return job, nil + } + } + return s.latest, nil +} +func (s *materializeJobRepoStub) GetByIdempotencyKey(key string) (*models.SkillPackageMaterializeJob, error) { + if s.byKey != nil { + if job, ok := s.byKey[key]; ok { + return job, nil + } + } + return nil, nil +} +func (s *materializeJobRepoStub) ClaimNextPending(context.Context, int) ([]models.SkillPackageMaterializeJob, error) { + return nil, nil +} +func (s *materializeJobRepoStub) MarkSucceeded(id int) error { + for _, job := range s.created { + if job.ID == id { + job.Status = MaterializeJobStatusSucceeded + } + } + return nil +} +func (s *materializeJobRepoStub) MarkFailed(id int, msg string, _ bool) error { + for _, job := range s.created { + if job.ID == id { + job.Status = MaterializeJobStatusFailed + job.LastError = &msg + } + } + if s.latest != nil && s.latest.ID == id { + s.latest.Status = MaterializeJobStatusFailed + s.latest.LastError = &msg + } + return nil +} +func (s *materializeJobRepoStub) MarkRunning(id int) error { + for _, job := range s.created { + if job.ID == id { + job.Status = MaterializeJobStatusRunning + } + } + if s.latest != nil && s.latest.ID == id { + s.latest.Status = MaterializeJobStatusRunning + } + return nil +} +func (s *materializeJobRepoStub) ReleaseToPending(int) error { return nil } +func (s *materializeJobRepoStub) ResetForRetry(int) error { return nil } +func (s *materializeJobRepoStub) RequeueExisting(id, blobID int, contentHash, workspaceDir string) error { + for _, job := range s.created { + if job.ID == id { + job.Status = MaterializeJobStatusPending + job.BlobID = blobID + job.ContentHash = contentHash + job.WorkspaceDir = workspaceDir + job.LastError = nil + } + } + if s.latest != nil && s.latest.ID == id { + s.latest.Status = MaterializeJobStatusPending + s.latest.BlobID = blobID + s.latest.ContentHash = contentHash + s.latest.WorkspaceDir = workspaceDir + s.latest.LastError = nil + } + return nil +} +func (s *materializeJobRepoStub) FindLatestBySkillID(int) (*models.SkillPackageMaterializeJob, error) { + return s.latest, nil +} +func (s *materializeJobRepoStub) CountPendingByInstance(int) (int, error) { return 0, nil } +func (s *materializeJobRepoStub) ListBackfillCandidates(int) ([]repository.SkillPackageMaterializeBackfillCandidate, error) { + return nil, nil +} + +type materializeBlobRepoStub struct { + blobs map[int]*models.SkillBlob +} + +func (s *materializeBlobRepoStub) GetBlobByID(id int) (*models.SkillBlob, error) { + if s.blobs == nil { + return nil, nil + } + return s.blobs[id], nil +} + +type testSkillScanner struct{} + +func (testSkillScanner) ScanArchive(context.Context, string, []byte, map[string]string) (string, map[string]interface{}, string, error) { + return skillRiskNone, map[string]interface{}{}, "ok", nil +} +func (testSkillScanner) AvailableAnalyzers(context.Context) ([]string, error) { return nil, nil } + +func TestProcessJobMaterializesFromWorkspaceWithStorage(t *testing.T) { + root := t.TempDir() + workspace := filepath.Join(root, "hermes", "user-1", "instance-1") + skillRoot := filepath.Join(workspace, "home", ".hermes", "skills", "demo") + if err := os.MkdirAll(filepath.Join(skillRoot, "src"), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# demo\n"), 0o640); err != nil { + t.Fatal(err) + } + + instance := &models.Instance{ + ID: 1, + UserID: 1, + Type: RuntimeTypeHermes, + InstanceMode: InstanceModeLite, + RuntimeType: RuntimeBackendGateway, + WorkspacePath: strPtr(workspace), + } + _, contentHash, err := loadLiteSkillDirectoryFromWorkspace(instance, "demo") + if err != nil { + t.Fatalf("loadLiteSkillDirectoryFromWorkspace() error = %v", err) + } + + blobID := 20 + stub := &skillRepoStub{ + skills: map[int]*models.Skill{ + 1: { + ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, + SourceType: skillSourceDiscovered, CurrentVersionID: intPtr(10), + }, + }, + versions: map[int]*models.SkillVersion{10: {ID: 10, SkillID: 1, BlobID: blobID}}, + blobs: map[int]*models.SkillBlob{ + blobID: {ID: blobID, ContentHash: contentHash, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: instance}} + storage := &importTestObjectStorage{objects: map[string][]byte{}} + svc := &skillService{ + repo: stub, + instanceRepo: instRepo, + storage: storage, + scanner: testSkillScanner{}, + } + job := &models.SkillPackageMaterializeJob{ + ID: 1, InstanceID: 1, SkillID: 1, BlobID: blobID, + WorkspaceDir: "demo", ContentHash: contentHash, Status: MaterializeJobStatusPending, + } + jobRepo := &materializeJobRepoStub{latest: job, created: []*models.SkillPackageMaterializeJob{job}} + matSvc := NewSkillPackageMaterializeService(jobRepo, stub, SkillServiceAsMaterializer(svc)) + + if err := matSvc.ProcessJob(context.Background(), job.ID); err != nil { + t.Fatalf("ProcessJob() error = %v", err) + } + if job.Status != MaterializeJobStatusSucceeded { + t.Fatalf("job status = %q, want %q", job.Status, MaterializeJobStatusSucceeded) + } + updated, err := stub.GetBlobByID(blobID) + if err != nil { + t.Fatal(err) + } + if updated == nil || strings.TrimSpace(updated.ObjectKey) == "" { + t.Fatalf("expected blob object key, got %#v", updated) + } + if !strings.EqualFold(strings.TrimSpace(updated.ScanStatus), "completed") { + t.Fatalf("blob scan_status = %q, want completed", updated.ScanStatus) + } + if _, ok := storage.objects[updated.ObjectKey]; !ok { + t.Fatalf("storage missing object %q", updated.ObjectKey) + } +} diff --git a/backend/internal/services/skill_package_materialize_worker.go b/backend/internal/services/skill_package_materialize_worker.go new file mode 100644 index 0000000..c1d077b --- /dev/null +++ b/backend/internal/services/skill_package_materialize_worker.go @@ -0,0 +1,140 @@ +package services + +import ( + "context" + "log" + "sync" + "time" +) + +type SkillPackageMaterializeWorker struct { + service *SkillPackageMaterializeService + tick time.Duration + batchSize int + concurrency int + perInstanceLimit int + enabled bool + + mu sync.Mutex + running bool + stopChan chan struct{} +} + +func NewSkillPackageMaterializeWorker(service *SkillPackageMaterializeService, tick time.Duration, batchSize, concurrency, perInstanceLimit int, enabled bool) *SkillPackageMaterializeWorker { + if tick <= 0 { + tick = 2 * time.Second + } + if batchSize <= 0 { + batchSize = 5 + } + if concurrency <= 0 { + concurrency = 5 + } + if perInstanceLimit <= 0 { + perInstanceLimit = 2 + } + return &SkillPackageMaterializeWorker{ + service: service, + tick: tick, + batchSize: batchSize, + concurrency: concurrency, + perInstanceLimit: perInstanceLimit, + enabled: enabled, + } +} + +func (w *SkillPackageMaterializeWorker) Start() { + if w == nil || !w.enabled || w.service == nil { + return + } + w.mu.Lock() + defer w.mu.Unlock() + if w.running { + return + } + w.stopChan = make(chan struct{}) + w.running = true + go w.loop(w.stopChan) +} + +func (w *SkillPackageMaterializeWorker) Stop() { + if w == nil { + return + } + w.mu.Lock() + defer w.mu.Unlock() + if !w.running { + return + } + close(w.stopChan) + w.running = false +} + +func (w *SkillPackageMaterializeWorker) loop(stop <-chan struct{}) { + ctx := context.Background() + if count, err := w.service.BackfillOnce(ctx, 500); err != nil { + log.Printf("skill package materialize backfill failed: %v", err) + } else if count > 0 { + log.Printf("skill package materialize backfill enqueued %d jobs", count) + } + + ticker := time.NewTicker(w.tick) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + w.processBatch(context.Background()) + } + } +} + +func (w *SkillPackageMaterializeWorker) processBatch(ctx context.Context) { + jobs, err := w.service.ClaimNextPending(ctx, w.batchSize) + if err != nil { + log.Printf("skill package materialize claim failed: %v", err) + return + } + if len(jobs) == 0 { + return + } + + sem := make(chan struct{}, w.concurrency) + instanceActive := make(map[int]int) + var instanceMu sync.Mutex + var wg sync.WaitGroup + + for _, job := range jobs { + instanceMu.Lock() + if instanceActive[job.InstanceID] >= w.perInstanceLimit { + instanceMu.Unlock() + if err := w.service.ReleaseToPending(job.ID); err != nil { + log.Printf("skill package materialize release job %d failed: %v", job.ID, err) + } + continue + } + instanceActive[job.InstanceID]++ + instanceMu.Unlock() + + job := job + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + defer func() { + instanceMu.Lock() + instanceActive[job.InstanceID]-- + if instanceActive[job.InstanceID] <= 0 { + delete(instanceActive, job.InstanceID) + } + instanceMu.Unlock() + }() + if err := w.service.ProcessJob(ctx, job.ID); err != nil { + log.Printf("skill package materialize job %d failed: %v", job.ID, err) + } + }() + } + wg.Wait() +} diff --git a/backend/internal/services/skill_package_materialize_worker_test.go b/backend/internal/services/skill_package_materialize_worker_test.go new file mode 100644 index 0000000..320922e --- /dev/null +++ b/backend/internal/services/skill_package_materialize_worker_test.go @@ -0,0 +1,76 @@ +package services + +import ( + "context" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +func TestSkillPackageMaterializeWorkerReleasesWhenInstanceLimitReached(t *testing.T) { + repo := &workerMaterializeJobRepoStub{ + jobs: []models.SkillPackageMaterializeJob{ + {ID: 1, InstanceID: 9, Status: MaterializeJobStatusRunning}, + {ID: 2, InstanceID: 9, Status: MaterializeJobStatusRunning}, + {ID: 3, InstanceID: 9, Status: MaterializeJobStatusRunning}, + }, + } + service := NewSkillPackageMaterializeService(repo, nil, stubMaterializer{}) + worker := NewSkillPackageMaterializeWorker(service, time.Second, 3, 5, 2, true) + + worker.processBatch(context.Background()) + + if repo.released != 1 { + t.Fatalf("released = %d, want 1", repo.released) + } +} + +type stubMaterializer struct{} + +func (stubMaterializer) materializeSkillPackageFromWorkspace(context.Context, int, string, string, int) (*models.SkillBlob, error) { + return &models.SkillBlob{ObjectKey: "discovered/1/demo/hash.zip", ScanStatus: "completed"}, nil +} + +func (stubMaterializer) syncSkillRecordFromBlob(int, *models.SkillBlob) error { return nil } + +type workerMaterializeJobRepoStub struct { + jobs []models.SkillPackageMaterializeJob + released int +} + +func (s *workerMaterializeJobRepoStub) Create(*models.SkillPackageMaterializeJob) error { return nil } +func (s *workerMaterializeJobRepoStub) GetByID(id int) (*models.SkillPackageMaterializeJob, error) { + for i := range s.jobs { + if s.jobs[i].ID == id { + job := s.jobs[i] + return &job, nil + } + } + return nil, nil +} +func (s *workerMaterializeJobRepoStub) GetByIdempotencyKey(string) (*models.SkillPackageMaterializeJob, error) { + return nil, nil +} +func (s *workerMaterializeJobRepoStub) ClaimNextPending(context.Context, int) ([]models.SkillPackageMaterializeJob, error) { + return s.jobs, nil +} +func (s *workerMaterializeJobRepoStub) MarkSucceeded(int) error { return nil } +func (s *workerMaterializeJobRepoStub) MarkFailed(int, string, bool) error { return nil } +func (s *workerMaterializeJobRepoStub) MarkRunning(int) error { return nil } +func (s *workerMaterializeJobRepoStub) ReleaseToPending(int) error { + s.released++ + return nil +} +func (s *workerMaterializeJobRepoStub) ResetForRetry(int) error { return nil } +func (s *workerMaterializeJobRepoStub) RequeueExisting(int, int, string, string) error { + return nil +} +func (s *workerMaterializeJobRepoStub) FindLatestBySkillID(int) (*models.SkillPackageMaterializeJob, error) { + return nil, nil +} +func (s *workerMaterializeJobRepoStub) CountPendingByInstance(int) (int, error) { return 0, nil } +func (s *workerMaterializeJobRepoStub) ListBackfillCandidates(int) ([]repository.SkillPackageMaterializeBackfillCandidate, error) { + return nil, nil +} diff --git a/backend/internal/services/skill_runtime_paths.go b/backend/internal/services/skill_runtime_paths.go new file mode 100644 index 0000000..65f3e76 --- /dev/null +++ b/backend/internal/services/skill_runtime_paths.go @@ -0,0 +1,171 @@ +package services + +import ( + "os" + "path/filepath" + "strings" + + "clawreef/internal/models" +) + +const runtimeSkillDiscoveryMaxDepth = 2 + +type runtimeSkillDiscovery struct { + RelativePath string + SkillRoot string +} + +func runtimeSkillInstallRoot(instance *models.Instance) string { + if instance == nil || instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" { + return "" + } + workspacePath := filepath.Clean(strings.TrimSpace(*instance.WorkspacePath)) + if isLiteRuntimeInstance(instance) { + if strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeHermes) { + return filepath.Join(workspacePath, "home", ".hermes", "skills") + } + return filepath.Join(workspacePath, "home", ".openclaw", "workspace", "skills") + } + if strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeHermes) { + return filepath.Join(workspacePath, ".hermes", "skills") + } + return filepath.Join(workspacePath, "home", ".openclaw", "workspace", "skills") +} + +func liteSkillInstallRoot(instance *models.Instance) string { + return runtimeSkillInstallRoot(instance) +} + +func sanitizeWorkspaceRelativePath(value string) string { + value = strings.TrimSpace(value) + value = strings.ReplaceAll(value, "\\", "/") + value = strings.Trim(value, "/") + if value == "" || strings.Contains(value, "..") { + return "" + } + parts := make([]string, 0, strings.Count(value, "/")+1) + for _, part := range strings.Split(value, "/") { + part = strings.TrimSpace(part) + if part == "" || part == "." || part == ".." || strings.HasPrefix(part, ".") { + return "" + } + parts = append(parts, part) + } + if len(parts) == 0 || len(parts) > runtimeSkillDiscoveryMaxDepth { + return "" + } + return strings.Join(parts, "/") +} + +func skillKeyFromRelativePath(relativePath string) string { + relativePath = sanitizeWorkspaceRelativePath(relativePath) + if relativePath == "" { + return "" + } + parts := strings.Split(relativePath, "/") + return sanitizeSkillKey(parts[len(parts)-1]) +} + +func runtimeSkillInstallRelativePath(instance *models.Instance, relativePath string) string { + relativePath = sanitizeWorkspaceRelativePath(relativePath) + if relativePath == "" { + return "" + } + if instance == nil || instance.WorkspacePath == nil { + return relativePath + } + workspacePath := filepath.Clean(strings.TrimSpace(*instance.WorkspacePath)) + target := filepath.Join(runtimeSkillInstallRoot(instance), filepath.FromSlash(relativePath)) + rel, err := filepath.Rel(workspacePath, target) + if err != nil { + return filepath.ToSlash(relativePath) + } + return filepath.ToSlash(rel) +} + +func joinRuntimeSkillPath(root, relativePath string) (string, error) { + root = filepath.Clean(strings.TrimSpace(root)) + relativePath = sanitizeWorkspaceRelativePath(relativePath) + if root == "" || relativePath == "" { + return "", filepath.ErrBadPattern + } + target := filepath.Join(root, filepath.FromSlash(relativePath)) + if !isPathWithin(root, target) { + return "", filepath.ErrBadPattern + } + return target, nil +} + +func discoverRuntimeSkillDirectories(root string, maxDepth int) ([]runtimeSkillDiscovery, error) { + root = filepath.Clean(strings.TrimSpace(root)) + if root == "" { + return nil, nil + } + if maxDepth <= 0 { + maxDepth = runtimeSkillDiscoveryMaxDepth + } + result := make([]runtimeSkillDiscovery, 0) + var walk func(currentRoot, relativePrefix string, depth int) error + walk = func(currentRoot, relativePrefix string, depth int) error { + entries, err := readDirNames(currentRoot) + if err != nil { + return err + } + for _, entry := range entries { + name := strings.TrimSpace(entry.Name) + if name == "" || strings.HasPrefix(name, ".") || name == ".tmp" { + continue + } + if !entry.IsDir { + continue + } + skillRoot := filepath.Join(currentRoot, name) + relativePath := name + if relativePrefix != "" { + relativePath = relativePrefix + "/" + name + } + relativePath = sanitizeWorkspaceRelativePath(relativePath) + if relativePath == "" { + continue + } + files, err := collectLiteSkillDirectoryFiles(skillRoot) + if err != nil { + return err + } + if len(files) > 0 { + result = append(result, runtimeSkillDiscovery{ + RelativePath: relativePath, + SkillRoot: skillRoot, + }) + continue + } + if depth < maxDepth { + if err := walk(skillRoot, relativePath, depth+1); err != nil { + return err + } + } + } + return nil + } + if err := walk(root, "", 1); err != nil { + return nil, err + } + return result, nil +} + +func readDirNames(path string) ([]dirEntryName, error) { + entries, err := os.ReadDir(path) + if err != nil { + return nil, err + } + result := make([]dirEntryName, 0, len(entries)) + for _, entry := range entries { + result = append(result, dirEntryName{Name: entry.Name(), IsDir: entry.IsDir()}) + } + return result, nil +} + +type dirEntryName struct { + Name string + IsDir bool +} diff --git a/backend/internal/services/skill_runtime_sync.go b/backend/internal/services/skill_runtime_sync.go new file mode 100644 index 0000000..a1a7258 --- /dev/null +++ b/backend/internal/services/skill_runtime_sync.go @@ -0,0 +1,356 @@ +package services + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +type runtimeSkillSyncDeps struct { + bindingRepo repository.InstanceRuntimeBindingRepository + runtimePodRepo repository.RuntimePodRepository + runtimeAgentClient RuntimeAgentClient +} + +func (s *skillService) ConfigureRuntimeSkillSync(bindingRepo repository.InstanceRuntimeBindingRepository, runtimePodRepo repository.RuntimePodRepository, agentClient RuntimeAgentClient) { + if s == nil { + return + } + s.runtimeSkillSync = &runtimeSkillSyncDeps{ + bindingRepo: bindingRepo, + runtimePodRepo: runtimePodRepo, + runtimeAgentClient: agentClient, + } +} + +func ConfigureSkillRuntimeSync(service SkillService, bindingRepo repository.InstanceRuntimeBindingRepository, runtimePodRepo repository.RuntimePodRepository, agentClient RuntimeAgentClient) { + if impl, ok := service.(*skillService); ok { + impl.ConfigureRuntimeSkillSync(bindingRepo, runtimePodRepo, agentClient) + } +} + +func (s *skillService) SyncRuntimeAgentSkillsReport(payload map[string]any) error { + if s == nil { + return fmt.Errorf("skill service is not configured") + } + reports, mode, reportedAt, err := parseRuntimeAgentSkillsReport(payload) + if err != nil { + return err + } + for _, report := range reports { + if report.InstanceID <= 0 { + continue + } + skills := make([]AgentSkillRecord, 0, len(report.Skills)) + for _, record := range report.Skills { + record.Source = normalizeRuntimeSkillSource(record.Source) + skills = append(skills, record) + } + req := AgentSkillInventoryReportRequest{ + AgentID: fmt.Sprintf("runtime-instance-%d", report.InstanceID), + ReportedAt: reportedAt, + Mode: mode, + Trigger: "runtime_agent_report", + Skills: skills, + } + if err := s.SyncAgentSkills(report.InstanceID, req); err != nil { + return err + } + s.completePendingSkillInventorySync(report.InstanceID) + } + return nil +} + +func (s *skillService) RequestLiteSkillInventorySync(instanceID int) error { + if s == nil || s.instanceRepo == nil { + return fmt.Errorf("skill service is not configured") + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return err + } + if instance == nil { + return fmt.Errorf("instance not found") + } + if err := EnsureInstanceWorkspacePathForServerScan(context.Background(), s.instanceRepo, instance); err != nil { + return err + } + if !isLiteRuntimeInstance(instance) && !SupportsServerWorkspaceSkillScan(instance) { + return fmt.Errorf("instance does not support workspace skill inventory sync") + } + + if SupportsServerWorkspaceSkillScan(instance) { + workspaceMode := "full" + willResyncAgent := isLiteRuntimeInstance(instance) && s.runtimeSkillSync != nil + if willResyncAgent { + workspaceMode = "incremental" + } + if err := s.syncRuntimeSkillsFromWorkspace(instanceID, workspaceMode); err != nil { + return err + } + if !willResyncAgent { + s.completePendingSkillInventorySync(instanceID) + } + } + + if !isLiteRuntimeInstance(instance) || s.runtimeSkillSync == nil { + return nil + } + deps := s.runtimeSkillSync + if deps.bindingRepo == nil || deps.runtimePodRepo == nil || deps.runtimeAgentClient == nil { + return nil + } + + ctx := context.Background() + binding, err := deps.bindingRepo.GetRunningByInstanceID(ctx, instanceID) + if err != nil { + return fmt.Errorf("failed to resolve runtime binding: %w", err) + } + if binding == nil { + binding, err = deps.bindingRepo.GetByInstanceID(ctx, instanceID) + if err != nil { + return fmt.Errorf("failed to resolve runtime binding: %w", err) + } + } + if binding == nil || binding.Generation != instance.RuntimeGeneration { + return nil + } + runtimePod, err := deps.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if err != nil { + return fmt.Errorf("failed to resolve runtime pod: %w", err) + } + if runtimePod != nil && runtimePod.AgentEndpoint != nil && strings.TrimSpace(*runtimePod.AgentEndpoint) != "" { + if err := deps.runtimeAgentClient.ResyncInstanceSkills(ctx, strings.TrimSpace(*runtimePod.AgentEndpoint), instanceID, "full"); err != nil { + return fmt.Errorf("failed to request runtime skill inventory resync: %w", err) + } + } + return nil +} + +func (s *skillService) syncRuntimeSkillsFromWorkspace(instanceID int, mode string) error { + if s == nil || s.instanceRepo == nil { + return fmt.Errorf("skill service is not configured") + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return err + } + if instance == nil { + return fmt.Errorf("instance not found") + } + if err := EnsureInstanceWorkspacePathForServerScan(context.Background(), s.instanceRepo, instance); err != nil { + return err + } + if !isLiteRuntimeInstance(instance) && !SupportsServerWorkspaceSkillScan(instance) { + return fmt.Errorf("instance does not support workspace skill inventory sync") + } + + root := runtimeSkillInstallRoot(instance) + if root == "" { + return fmt.Errorf("runtime skill workspace root is not configured") + } + + records := make([]AgentSkillRecord, 0) + if _, err := os.Stat(root); err != nil { + if os.IsNotExist(err) { + return s.SyncAgentSkills(instanceID, runtimeWorkspaceSkillInventoryRequest(instanceID, mode, records)) + } + return fmt.Errorf("failed to inspect runtime skill directory: %w", err) + } + discoveries, err := discoverRuntimeSkillDirectories(root, runtimeSkillDiscoveryMaxDepth) + if err != nil { + return fmt.Errorf("failed to scan runtime skill directory: %w", err) + } + + for _, discovery := range discoveries { + files, err := collectLiteSkillDirectoryFiles(discovery.SkillRoot) + if err != nil { + return fmt.Errorf("failed to scan runtime skill %q: %w", discovery.RelativePath, err) + } + if len(files) == 0 { + continue + } + records = append(records, AgentSkillRecord{ + Identifier: discovery.RelativePath, + InstallPath: runtimeSkillInstallRelativePath(instance, discovery.RelativePath), + ContentMD5: hashDirectory(files), + Source: "discovered_in_instance", + Type: "agent-skill", + }) + } + + return s.SyncAgentSkills(instanceID, runtimeWorkspaceSkillInventoryRequest(instanceID, mode, records)) +} + +func (s *skillService) syncLiteSkillsFromWorkspace(instanceID int) error { + return s.syncRuntimeSkillsFromWorkspace(instanceID, "full") +} + +func runtimeWorkspaceSkillInventoryRequest(instanceID int, mode string, records []AgentSkillRecord) AgentSkillInventoryReportRequest { + now := time.Now().UTC() + normalizedMode := strings.TrimSpace(mode) + if normalizedMode == "" { + normalizedMode = "full" + } + return AgentSkillInventoryReportRequest{ + AgentID: fmt.Sprintf("workspace-scan-instance-%d", instanceID), + ReportedAt: &now, + Mode: normalizedMode, + Trigger: "runtime_workspace_scan", + Skills: records, + } +} + +func liteWorkspaceSkillInventoryRequest(instanceID int, records []AgentSkillRecord) AgentSkillInventoryReportRequest { + return runtimeWorkspaceSkillInventoryRequest(instanceID, "full", records) +} + +func liteSkillInstallRelativePath(instance *models.Instance, skillName string) string { + return runtimeSkillInstallRelativePath(instance, skillName) +} + +func collectLiteSkillDirectoryFiles(skillRoot string) (map[string][]byte, error) { + manifestPath := filepath.Join(skillRoot, "SKILL.md") + info, err := os.Stat(manifestPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if info.IsDir() { + return nil, nil + } + + files := map[string][]byte{} + err = filepath.WalkDir(skillRoot, func(current string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return nil + } + rel, err := filepath.Rel(skillRoot, current) + if err != nil { + return err + } + rel = normalizeSkillRelPath(filepath.ToSlash(rel)) + if rel == "" || hasHiddenPathSegment(rel) { + return nil + } + body, err := os.ReadFile(current) + if err != nil { + return err + } + files[rel] = body + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} + +type runtimeAgentInstanceSkillReport struct { + InstanceID int + Skills []AgentSkillRecord +} + +func parseRuntimeAgentSkillsReport(payload map[string]any) ([]runtimeAgentInstanceSkillReport, string, *time.Time, error) { + if payload == nil { + return nil, "", nil, fmt.Errorf("skills report payload is required") + } + raw, err := json.Marshal(payload) + if err != nil { + return nil, "", nil, fmt.Errorf("failed to encode skills report payload: %w", err) + } + var decoded struct { + Mode string `json:"mode"` + ReportedAt *time.Time `json:"reported_at"` + Instances []struct { + InstanceID int `json:"instance_id"` + Skills []AgentSkillRecord `json:"skills"` + } `json:"instances"` + } + if err := json.Unmarshal(raw, &decoded); err != nil { + return nil, "", nil, fmt.Errorf("failed to decode skills report payload: %w", err) + } + mode := strings.TrimSpace(decoded.Mode) + if mode == "" { + mode = "full" + } + reports := make([]runtimeAgentInstanceSkillReport, 0, len(decoded.Instances)) + for _, item := range decoded.Instances { + reports = append(reports, runtimeAgentInstanceSkillReport{ + InstanceID: item.InstanceID, + Skills: item.Skills, + }) + } + return reports, mode, decoded.ReportedAt, nil +} + +func normalizeRuntimeSkillSource(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "runtime", "discovered", "agent-skill", "agent_skill": + return "discovered_in_instance" + default: + return normalizeSkillSource(value) + } +} + +func (s *skillService) CompletePendingSkillInventorySync(instanceID int) { + s.completePendingSkillInventorySync(instanceID) +} + +func (s *skillService) completePendingSkillInventorySync(instanceID int) { + if s == nil || s.commandRepo == nil || instanceID <= 0 { + return + } + commands, err := s.commandRepo.ListByInstanceID(instanceID, 20) + if err != nil { + return + } + now := time.Now().UTC() + for _, command := range commands { + if command.CommandType != InstanceCommandTypeSyncSkillInventory { + continue + } + switch strings.TrimSpace(command.Status) { + case instanceCommandStatusPending, instanceCommandStatusDispatched, instanceCommandStatusRunning: + command.Status = instanceCommandStatusSucceeded + command.FinishedAt = &now + command.UpdatedAt = now + _ = s.commandRepo.Update(&command) + return + } + } +} + +func SupportsServerWorkspaceSkillScan(instance *models.Instance) bool { + if instance == nil || instance.WorkspacePath == nil { + return false + } + if strings.TrimSpace(*instance.WorkspacePath) == "" { + return false + } + switch strings.ToLower(strings.TrimSpace(instance.Type)) { + case "hermes", "openclaw": + return true + default: + return false + } +} + +func IsLiteRuntimeInstance(instance *models.Instance) bool { + return isLiteRuntimeInstance(instance) +} diff --git a/backend/internal/services/skill_runtime_sync_test.go b/backend/internal/services/skill_runtime_sync_test.go new file mode 100644 index 0000000..8ff9f05 --- /dev/null +++ b/backend/internal/services/skill_runtime_sync_test.go @@ -0,0 +1,472 @@ +package services + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "clawreef/internal/models" +) + +func TestParseRuntimeAgentSkillsReport(t *testing.T) { + payload := map[string]any{ + "mode": "full", + "instances": []map[string]any{ + { + "instance_id": 12, + "skills": []map[string]any{ + { + "identifier": "weather", + "content_md5": "abc123", + "source": "runtime", + }, + }, + }, + }, + } + reports, mode, _, err := parseRuntimeAgentSkillsReport(payload) + if err != nil { + t.Fatalf("parseRuntimeAgentSkillsReport() error = %v", err) + } + if mode != "full" { + t.Fatalf("mode = %q, want full", mode) + } + if len(reports) != 1 || reports[0].InstanceID != 12 { + t.Fatalf("unexpected reports: %#v", reports) + } + if len(reports[0].Skills) != 1 || reports[0].Skills[0].Identifier != "weather" { + t.Fatalf("unexpected skills: %#v", reports[0].Skills) + } +} + +func TestNormalizeRuntimeSkillSource(t *testing.T) { + if got := normalizeRuntimeSkillSource("runtime"); got != "discovered_in_instance" { + t.Fatalf("normalizeRuntimeSkillSource(runtime) = %q", got) + } + if got := normalizeRuntimeSkillSource("injected_by_clawmanager"); got != "injected_by_clawmanager" { + t.Fatalf("normalizeRuntimeSkillSource(injected) = %q", got) + } +} + +func TestCollectLiteSkillDirectoryFiles(t *testing.T) { + root := t.TempDir() + skillRoot := filepath.Join(root, "weather") + if err := os.MkdirAll(filepath.Join(skillRoot, "src"), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# weather\n"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillRoot, "src", "main.py"), []byte("print('ok')\n"), 0o640); err != nil { + t.Fatal(err) + } + + files, err := collectLiteSkillDirectoryFiles(skillRoot) + if err != nil { + t.Fatalf("collectLiteSkillDirectoryFiles() error = %v", err) + } + if len(files) != 2 { + t.Fatalf("files = %#v, want 2 entries", files) + } + if got := hashDirectory(files); got == "" { + t.Fatal("expected non-empty content md5") + } +} + +func TestCollectLiteSkillDirectoryFilesSkipsWithoutManifest(t *testing.T) { + root := t.TempDir() + skillRoot := filepath.Join(root, "orphan") + if err := os.MkdirAll(skillRoot, 0o750); err != nil { + t.Fatal(err) + } + files, err := collectLiteSkillDirectoryFiles(skillRoot) + if err != nil { + t.Fatalf("collectLiteSkillDirectoryFiles() error = %v", err) + } + if files != nil { + t.Fatalf("files = %#v, want nil", files) + } +} + +func TestDiscoverRuntimeSkillDirectoriesNestedCategory(t *testing.T) { + root := t.TempDir() + categoryRoot := filepath.Join(root, "productivity") + skillRoot := filepath.Join(categoryRoot, "my-skill") + for _, dir := range []string{skillRoot, filepath.Join(skillRoot, "src")} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# my skill\n"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillRoot, "src", "main.py"), []byte("print('ok')\n"), 0o640); err != nil { + t.Fatal(err) + } + + discoveries, err := discoverRuntimeSkillDirectories(root, runtimeSkillDiscoveryMaxDepth) + if err != nil { + t.Fatalf("discoverRuntimeSkillDirectories() error = %v", err) + } + if len(discoveries) != 1 { + t.Fatalf("discoveries = %#v, want 1 nested skill", discoveries) + } + if discoveries[0].RelativePath != "productivity/my-skill" { + t.Fatalf("RelativePath = %q, want productivity/my-skill", discoveries[0].RelativePath) + } +} + +func TestDiscoverRuntimeSkillDirectoriesFlatAndNested(t *testing.T) { + root := t.TempDir() + flatRoot := filepath.Join(root, "weather") + if err := os.MkdirAll(flatRoot, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(flatRoot, "SKILL.md"), []byte("# weather\n"), 0o640); err != nil { + t.Fatal(err) + } + + discoveries, err := discoverRuntimeSkillDirectories(root, runtimeSkillDiscoveryMaxDepth) + if err != nil { + t.Fatalf("discoverRuntimeSkillDirectories() error = %v", err) + } + if len(discoveries) != 1 || discoveries[0].RelativePath != "weather" { + t.Fatalf("discoveries = %#v, want flat weather skill", discoveries) + } +} + +func TestRuntimeSkillInstallRootOpenClawAndHermes(t *testing.T) { + workspace := "/workspaces/demo/instance-1" + hermes := &models.Instance{Type: RuntimeTypeHermes, WorkspacePath: &workspace} + openclaw := &models.Instance{Type: RuntimeTypeOpenClaw, WorkspacePath: &workspace} + + hermesRoot := runtimeSkillInstallRoot(hermes) + openclawRoot := runtimeSkillInstallRoot(openclaw) + if hermesRoot != filepath.Join(workspace, ".hermes", "skills") { + t.Fatalf("hermes root = %q", hermesRoot) + } + if openclawRoot != filepath.Join(workspace, "home", ".openclaw", "workspace", "skills") { + t.Fatalf("openclaw root = %q", openclawRoot) + } +} + +func TestResolveInstanceSkillSourceTypePreservesInjected(t *testing.T) { + existing := &models.InstanceSkill{SourceType: "injected_by_clawmanager"} + skill := &models.Skill{SourceType: skillSourceUploaded} + got := resolveInstanceSkillSourceType(existing, "discovered_in_instance", skill) + if got != "injected_by_clawmanager" { + t.Fatalf("resolveInstanceSkillSourceType() = %q, want injected_by_clawmanager", got) + } +} + +func TestSanitizeWorkspaceRelativePath(t *testing.T) { + if got := sanitizeWorkspaceRelativePath("productivity/my-skill"); got != "productivity/my-skill" { + t.Fatalf("sanitizeWorkspaceRelativePath() = %q", got) + } + if got := sanitizeWorkspaceRelativePath("../escape"); got != "" { + t.Fatalf("sanitizeWorkspaceRelativePath(../escape) = %q, want empty", got) + } +} + +func TestIsLiteRuntimeInstanceOpenClawVariants(t *testing.T) { + liteGateway := &models.Instance{InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway, Type: RuntimeTypeOpenClaw} + if !IsLiteRuntimeInstance(liteGateway) { + t.Fatal("expected gateway lite instance") + } + proDesktop := &models.Instance{InstanceMode: InstanceModePro, RuntimeType: RuntimeBackendDesktop, Type: RuntimeTypeOpenClaw} + if IsLiteRuntimeInstance(proDesktop) { + t.Fatal("expected pro desktop instance to be non-lite") + } + shellPod := &models.Instance{InstanceMode: InstanceModePro, RuntimeType: RuntimeBackendShell, Type: RuntimeTypeOpenClaw} + if IsLiteRuntimeInstance(shellPod) { + t.Fatal("expected shell pod instance to use pro agent path") + } +} + +type provenanceCaptureRepoStub struct { + capturingSkillRepoStub + upserted []*models.InstanceSkill +} + +func (s *provenanceCaptureRepoStub) GetBlobByContentHash(hash string) (*models.SkillBlob, error) { + for _, blob := range s.blobs { + if strings.EqualFold(strings.TrimSpace(blob.ContentHash), strings.TrimSpace(hash)) { + copy := *blob + return ©, nil + } + } + return nil, nil +} + +func (s *provenanceCaptureRepoStub) GetVersionBySkillAndBlob(skillID, blobID int) (*models.SkillVersion, error) { + for _, version := range s.versions { + if version.SkillID == skillID && version.BlobID == blobID { + copy := *version + return ©, nil + } + } + return nil, nil +} + +func (s *provenanceCaptureRepoStub) UpsertInstanceSkill(item *models.InstanceSkill) error { + copy := *item + s.upserted = append(s.upserted, ©) + updated := false + for i, existing := range s.instanceSkills { + if existing.InstanceID == item.InstanceID && existing.SkillID == item.SkillID { + s.instanceSkills[i] = copy + updated = true + break + } + } + if !updated { + s.instanceSkills = append(s.instanceSkills, copy) + } + return nil +} + +func TestSyncAgentSkillsPreservesInjectedProvenanceAfterWorkspaceScan(t *testing.T) { + contentHash := "abc123def456789012345678901234" + versionID := 1 + stub := &provenanceCaptureRepoStub{ + capturingSkillRepoStub: capturingSkillRepoStub{ + skillRepoStub: skillRepoStub{ + skills: map[int]*models.Skill{ + 10: { + ID: 10, UserID: 1, SkillKey: "ppt-1-0-0", Name: "ppt-1.0.0", + SourceType: skillSourceUploaded, Status: skillStatusActive, + Visibility: skillVisibilityPublic, CurrentVersionID: &versionID, + }, + }, + blobs: map[int]*models.SkillBlob{ + 1: {ID: 1, ContentHash: contentHash, ObjectKey: "hub/ppt.zip", ScanStatus: "completed"}, + }, + versions: map[int]*models.SkillVersion{ + 1: {ID: 1, SkillID: 10, BlobID: 1, VersionNo: 1}, + }, + instanceSkills: []models.InstanceSkill{ + {InstanceID: 1, SkillID: 10, SourceType: "injected_by_clawmanager", Status: "active"}, + }, + }, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{ + 1: {ID: 1, UserID: 1, Type: RuntimeTypeHermes, InstanceMode: InstanceModePro, RuntimeType: RuntimeBackendDesktop}, + }} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{ + Mode: "full", + Skills: []AgentSkillRecord{{ + Identifier: "ppt-1-0-0", + ContentMD5: contentHash, + Source: "discovered_in_instance", + InstallPath: "home/.hermes/skills/ppt-1-0-0", + }}, + }) + if err != nil { + t.Fatalf("SyncAgentSkills() error = %v", err) + } + if len(stub.upserted) != 1 { + t.Fatalf("upserted %d instance skills, want 1", len(stub.upserted)) + } + if stub.upserted[0].SourceType != "injected_by_clawmanager" { + t.Fatalf("SourceType = %q, want injected_by_clawmanager", stub.upserted[0].SourceType) + } +} + +func TestSyncAgentSkillsReusesUploadedSkillOnWorkspaceScan(t *testing.T) { + contentHash := "abc123def456789012345678901234" + versionID := 1 + stub := &provenanceCaptureRepoStub{ + capturingSkillRepoStub: capturingSkillRepoStub{ + skillRepoStub: skillRepoStub{ + skills: map[int]*models.Skill{ + 10: { + ID: 10, UserID: 1, SkillKey: "ppt-1-0-0", Name: "ppt-1.0.0", + SourceType: skillSourceUploaded, Status: skillStatusActive, + Visibility: skillVisibilityPublic, CurrentVersionID: &versionID, + }, + }, + blobs: map[int]*models.SkillBlob{ + 1: {ID: 1, ContentHash: contentHash, ObjectKey: "hub/ppt.zip", ScanStatus: "completed"}, + }, + versions: map[int]*models.SkillVersion{ + 1: {ID: 1, SkillID: 10, BlobID: 1, VersionNo: 1}, + }, + }, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{ + 1: {ID: 1, UserID: 1, Type: RuntimeTypeHermes, InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway}, + }} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{ + Mode: "full", + Skills: []AgentSkillRecord{{ + Identifier: "ppt-1-0-0", + ContentMD5: contentHash, + Source: "discovered_in_instance", + InstallPath: "home/.hermes/skills/ppt-1-0-0", + }}, + }) + if err != nil { + t.Fatalf("SyncAgentSkills() error = %v", err) + } + if len(stub.createdSkills) != 0 { + t.Fatalf("created %d discovered skills, want 0 reuse of uploaded skill", len(stub.createdSkills)) + } + if len(stub.upserted) != 1 || stub.upserted[0].SkillID != 10 { + t.Fatalf("upserted = %#v, want instance skill for uploaded skill id 10", stub.upserted) + } +} + +type recordingSkillResyncAgentClient struct { + fakeRuntimeAgentClient + calls []struct { + instanceID int + mode string + } + err error +} + +func (c *recordingSkillResyncAgentClient) ResyncInstanceSkills(_ context.Context, _ string, instanceID int, mode string) error { + c.calls = append(c.calls, struct { + instanceID int + mode string + }{instanceID: instanceID, mode: mode}) + return c.err +} + +type inventorySyncCommandRepo struct { + commands []models.InstanceCommand +} + +func (r *inventorySyncCommandRepo) Create(*models.InstanceCommand) error { return nil } +func (r *inventorySyncCommandRepo) Update(command *models.InstanceCommand) error { + for i := range r.commands { + if r.commands[i].ID == command.ID { + r.commands[i] = *command + return nil + } + } + r.commands = append(r.commands, *command) + return nil +} +func (r *inventorySyncCommandRepo) GetByID(int) (*models.InstanceCommand, error) { return nil, nil } +func (r *inventorySyncCommandRepo) GetByInstanceIdempotencyKey(int, string) (*models.InstanceCommand, error) { + return nil, nil +} +func (r *inventorySyncCommandRepo) GetNextPendingByInstance(int) (*models.InstanceCommand, error) { + return nil, nil +} +func (r *inventorySyncCommandRepo) ListByInstanceID(int, int) ([]models.InstanceCommand, error) { + return append([]models.InstanceCommand(nil), r.commands...), nil +} +func (r *inventorySyncCommandRepo) FindLatestFailedCollectSkillPackage(string) (*models.InstanceCommand, error) { + return nil, nil +} + +func TestRequestLiteSkillInventorySyncUsesIncrementalWhenAgentResyncFollows(t *testing.T) { + workspace := t.TempDir() + skillsRoot := filepath.Join(workspace, "home", ".hermes", "skills") + if err := os.MkdirAll(skillsRoot, 0o750); err != nil { + t.Fatal(err) + } + + stub := &capturingSkillRepoStub{ + skillRepoStub: skillRepoStub{ + skills: map[int]*models.Skill{ + 10: {ID: 10, UserID: 1, SkillKey: "weather", Name: "weather", SourceType: skillSourceDiscovered, Status: skillStatusActive}, + }, + blobs: map[int]*models.SkillBlob{}, + versions: map[int]*models.SkillVersion{}, + instanceSkills: []models.InstanceSkill{ + {InstanceID: 1, SkillID: 10, Status: "active", SourceType: "discovered_in_instance"}, + }, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{ + 1: { + ID: 1, UserID: 1, Type: RuntimeTypeHermes, InstanceMode: InstanceModeLite, + RuntimeType: RuntimeBackendGateway, WorkspacePath: &workspace, RuntimeGeneration: 1, + }, + }} + cmdRepo := &inventorySyncCommandRepo{commands: []models.InstanceCommand{{ + ID: 44, InstanceID: 1, CommandType: InstanceCommandTypeSyncSkillInventory, Status: instanceCommandStatusPending, + }}} + agent := &recordingSkillResyncAgentClient{} + endpoint := "http://runtime-agent" + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[1] = &models.InstanceRuntimeBinding{ + InstanceID: 1, RuntimePodID: 9, State: "running", Generation: 1, + } + podRepo := &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, AgentEndpoint: &endpoint}, + }} + + svc := &skillService{ + repo: stub, + instanceRepo: instRepo, + commandRepo: cmdRepo, + commandService: &noopInstanceCommandService{}, + } + svc.ConfigureRuntimeSkillSync(bindingRepo, podRepo, agent) + + if err := svc.RequestLiteSkillInventorySync(1); err != nil { + t.Fatalf("RequestLiteSkillInventorySync() error = %v", err) + } + if stub.markMissingCalls != 0 { + t.Fatalf("markMissingCalls = %d, want 0 when workspace sync is incremental", stub.markMissingCalls) + } + if stub.instanceSkills[0].Status != "active" { + t.Fatalf("status = %q, want active", stub.instanceSkills[0].Status) + } + if len(agent.calls) != 1 || agent.calls[0].mode != "full" { + t.Fatalf("resync calls = %#v, want one full resync", agent.calls) + } + if cmdRepo.commands[0].Status != instanceCommandStatusPending { + t.Fatalf("command status = %q, want pending until agent inventory arrives", cmdRepo.commands[0].Status) + } +} + +func TestRequestLiteSkillInventorySyncPropagatesResyncError(t *testing.T) { + workspace := t.TempDir() + skillsRoot := filepath.Join(workspace, "home", ".hermes", "skills") + if err := os.MkdirAll(skillsRoot, 0o750); err != nil { + t.Fatal(err) + } + + stub := &capturingSkillRepoStub{ + skillRepoStub: skillRepoStub{ + skills: map[int]*models.Skill{}, + blobs: map[int]*models.SkillBlob{}, + versions: map[int]*models.SkillVersion{}, + instanceSkills: nil, + }, + } + instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{ + 1: { + ID: 1, UserID: 1, Type: RuntimeTypeHermes, InstanceMode: InstanceModeLite, + RuntimeType: RuntimeBackendGateway, WorkspacePath: &workspace, RuntimeGeneration: 1, + }, + }} + agent := &recordingSkillResyncAgentClient{err: errors.New("resync failed")} + endpoint := "http://runtime-agent" + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[1] = &models.InstanceRuntimeBinding{ + InstanceID: 1, RuntimePodID: 9, State: "running", Generation: 1, + } + podRepo := &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, AgentEndpoint: &endpoint}, + }} + svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}} + svc.ConfigureRuntimeSkillSync(bindingRepo, podRepo, agent) + + err := svc.RequestLiteSkillInventorySync(1) + if err == nil || !strings.Contains(err.Error(), "resync") { + t.Fatalf("error = %v, want resync failure", err) + } +} diff --git a/backend/internal/services/skill_service.go b/backend/internal/services/skill_service.go index addbca8..d096454 100644 --- a/backend/internal/services/skill_service.go +++ b/backend/internal/services/skill_service.go @@ -21,6 +21,7 @@ import ( "clawreef/internal/models" "clawreef/internal/repository" + "clawreef/internal/utils" ) var ( @@ -37,6 +38,9 @@ const ( skillSourceUploaded = "uploaded" skillSourceDiscovered = "discovered" + + skillStatusActive = "active" + skillStatusDeleted = "deleted" ) type SkillPayload struct { @@ -59,6 +63,16 @@ type SkillPayload struct { RiskReason *string `json:"risk_reason,omitempty"` TopFindings []SkillFindingPayload `json:"top_findings,omitempty"` InstanceCount int `json:"instance_count"` + Visibility string `json:"visibility"` + PublishedAt *time.Time `json:"published_at,omitempty"` + PublishedBy *int `json:"published_by,omitempty"` + Tags []SkillHubTagPayload `json:"tags,omitempty"` + Publishable bool `json:"publishable"` + PublishBlockedReason *string `json:"publish_blocked_reason,omitempty"` + PackageCollectError *string `json:"package_collect_error,omitempty"` + PackageMaterializeStatus *string `json:"package_materialize_status,omitempty"` + PackageMaterializeError *string `json:"package_materialize_error,omitempty"` + OwnerUsername *string `json:"owner_username,omitempty"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } @@ -99,6 +113,7 @@ type InstanceSkillPayload struct { SkillVersionID *int `json:"skill_version_id,omitempty"` SourceType string `json:"source_type"` InstallPath *string `json:"install_path,omitempty"` + WorkspaceDir *string `json:"workspace_dir,omitempty"` ObservedHash *string `json:"observed_hash,omitempty"` ContentMD5 *string `json:"content_md5,omitempty"` Status string `json:"status"` @@ -165,58 +180,70 @@ type SkillService interface { ListSkills(userID int) ([]SkillPayload, error) ListAllSkills() ([]SkillPayload, error) ListAvailableSkillsForInstance(instanceID int, userID int, userRole string) ([]SkillPayload, error) - GetSkill(userID, skillID int) (*SkillPayload, error) + GetSkill(actorUserID int, actorRole string, skillID int) (*SkillPayload, error) UpdateSkill(userID, skillID int, req UpdateSkillRequest) (*SkillPayload, error) - DeleteSkill(userID, skillID int) error - DownloadSkill(userID, skillID int) ([]byte, string, error) + DeleteSkill(actorUserID int, actorRole string, skillID int) error + DownloadSkill(actorUserID int, actorRole string, skillID int) ([]byte, string, error) DownloadSkillVersionByExternalID(externalVersionID string) ([]byte, string, error) - ListVersions(userID, skillID int) ([]SkillVersionPayload, error) + ListVersions(actorUserID int, actorRole string, skillID int) ([]SkillVersionPayload, error) ListInstanceSkills(instanceID int) ([]InstanceSkillPayload, error) - AttachSkillToInstance(instanceID int, skillID int) (*InstanceSkillPayload, error) - AttachSkillToInstanceForActor(instanceID int, skillID int, userID int, userRole string) (*InstanceSkillPayload, error) + AttachSkillToInstance(actorUserID int, actorRole string, instanceID int, skillID int) (*InstanceSkillPayload, error) RemoveSkillFromInstance(instanceID int, skillID int) error SyncAgentSkills(instanceID int, req AgentSkillInventoryReportRequest) error UploadAgentSkillPackage(ctx context.Context, instanceID int, req AgentSkillPackageUploadRequest, fileHeader *multipart.FileHeader) (*SkillPayload, error) - ListScanResults(userID, skillID int) ([]SkillScanResultPayload, error) + ListScanResults(actorUserID int, actorRole string, skillID int) ([]SkillScanResultPayload, error) + ListHubTags(actorRole string) ([]SkillHubTagPayload, error) + ListHubCatalog(actorUserID int, actorRole string, query SkillHubCatalogQuery) (*SkillHubCatalogResponse, error) + ListMyHubSkills(userID int) ([]SkillPayload, error) + ListAllHubSkillsAdmin() ([]SkillPayload, error) + GetSkillHubDetail(actorUserID int, actorRole string, skillID int) (*SkillPayload, error) + PublishToHub(actorUserID int, actorRole string, skillID int, tagIDs []int) (*SkillPayload, error) + UnpublishFromHub(actorUserID int, actorRole string, skillID int) (*SkillPayload, error) + UpdateHubTags(actorUserID int, actorRole string, skillID int, tagIDs []int) (*SkillPayload, error) + InstallHubSkill(actorUserID int, actorRole string, skillID, instanceID int) (*InstanceSkillPayload, error) + PublishFromInstance(actorUserID int, actorRole string, instanceID, skillID int, tagIDs []int) (*SkillPayload, error) + ImportInstanceSkillToLibrary(actorUserID int, actorRole string, instanceID, skillID int) (*SkillPayload, error) + RetrySkillPackageCollection(actorUserID int, actorRole string, instanceID, skillID int) error + ListAttachableSkills(actorUserID int, actorRole string) ([]SkillPayload, error) + ImportHubArchive(ctx context.Context, userID int, fileHeader *multipart.FileHeader) ([]SkillPayload, error) + PreviewHubImport(ctx context.Context, userID int, fileHeader *multipart.FileHeader) ([]SkillImportPreviewItem, error) + ImportHubArchiveWithDecisions(ctx context.Context, userID int, fileHeader *multipart.FileHeader, decisions []SkillImportDecision) ([]SkillImportResultItem, error) + SyncRuntimeAgentSkillsReport(payload map[string]any) error + RequestLiteSkillInventorySync(instanceID int) error + CompletePendingSkillInventorySync(instanceID int) } type skillService struct { - repo repository.SkillRepository - instanceRepo repository.InstanceRepository - commandService InstanceCommandService - storage ObjectStorageService - scanner SkillScannerClient + repo repository.SkillRepository + instanceRepo repository.InstanceRepository + userRepo repository.UserRepository + commandService InstanceCommandService + commandRepo repository.InstanceCommandRepository + storage ObjectStorageService + scanner SkillScannerClient + runtimeSkillSync *runtimeSkillSyncDeps + materializeService *SkillPackageMaterializeService } -func NewSkillService(repo repository.SkillRepository, instanceRepo repository.InstanceRepository, commandService InstanceCommandService, storage ObjectStorageService, scanner SkillScannerClient) SkillService { - return &skillService{repo: repo, instanceRepo: instanceRepo, commandService: commandService, storage: storage, scanner: scanner} +func NewSkillService(repo repository.SkillRepository, instanceRepo repository.InstanceRepository, userRepo repository.UserRepository, commandService InstanceCommandService, commandRepo repository.InstanceCommandRepository, storage ObjectStorageService, scanner SkillScannerClient) SkillService { + return &skillService{repo: repo, instanceRepo: instanceRepo, userRepo: userRepo, commandService: commandService, commandRepo: commandRepo, storage: storage, scanner: scanner} +} + +func ConfigureSkillPackageMaterialize(service SkillService, materialize *SkillPackageMaterializeService) { + if impl, ok := service.(*skillService); ok { + impl.materializeService = materialize + } } func (s *skillService) ImportArchive(ctx context.Context, userID int, fileHeader *multipart.FileHeader) ([]SkillPayload, error) { - if !strings.HasSuffix(strings.ToLower(strings.TrimSpace(fileHeader.Filename)), ".zip") { - return nil, fmt.Errorf("only .zip skill archives are supported") - } - file, err := fileHeader.Open() - if err != nil { - return nil, fmt.Errorf("failed to open uploaded archive: %w", err) - } - defer file.Close() - - raw, err := io.ReadAll(file) - if err != nil { - return nil, fmt.Errorf("failed to read uploaded archive: %w", err) - } - directories, err := extractSkillDirectories(fileHeader.Filename, raw) + directories, filename, err := readSkillArchiveDirectories(fileHeader) if err != nil { return nil, err } - if len(directories) == 0 { - return nil, fmt.Errorf("no skill directories found in archive") - } results := make([]SkillPayload, 0, len(directories)) for _, dir := range directories { - payload, err := s.importDirectory(ctx, userID, dir, fileHeader.Filename) + payload, err := s.importDirectory(ctx, userID, dir, filename) if err != nil { return nil, err } @@ -232,11 +259,14 @@ func (s *skillService) ListSkills(userID int) ([]SkillPayload, error) { } filtered := make([]models.Skill, 0, len(items)) for _, item := range items { + if isDeletedSkill(&item) { + continue + } if isUserManagedSkill(item) { filtered = append(filtered, item) } } - return s.toSkillPayloads(filtered) + return s.toSkillPayloads(filtered, userID, "") } func (s *skillService) ListAllSkills() ([]SkillPayload, error) { @@ -244,7 +274,7 @@ func (s *skillService) ListAllSkills() ([]SkillPayload, error) { if err != nil { return nil, err } - return s.toSkillPayloads(items) + return s.toSkillPayloads(items, 0, "admin") } func (s *skillService) ListAvailableSkillsForInstance(instanceID int, userID int, userRole string) ([]SkillPayload, error) { @@ -274,20 +304,28 @@ func (s *skillService) ListAvailableSkillsForInstance(instanceID int, userID int } filtered = append(filtered, item) } - return s.toSkillPayloads(filtered) + return s.toSkillPayloads(filtered, userID, userRole) } -func (s *skillService) GetSkill(userID, skillID int) (*SkillPayload, error) { + +func (s *skillService) GetSkill(actorUserID int, actorRole string, skillID int) (*SkillPayload, error) { item, err := s.repo.GetSkillByID(skillID) if err != nil { return nil, err } - if item == nil || item.UserID != userID { + if item == nil || !s.CanViewSkill(actorUserID, actorRole, item) { return nil, fmt.Errorf("skill not found") } - if !isUserManagedSkill(*item) { + if !isUserManagedSkill(*item) && !isAdminRole(actorRole) && item.UserID != actorUserID { return nil, fmt.Errorf("skill not found") } - return s.toSkillPayload(*item) + payload, err := s.toSkillPayload(*item) + if err != nil { + return nil, err + } + if err := s.enrichSkillPayload(payload, *item, nil); err != nil { + return nil, err + } + return payload, nil } func (s *skillService) UpdateSkill(userID, skillID int, req UpdateSkillRequest) (*SkillPayload, error) { @@ -313,29 +351,39 @@ func (s *skillService) UpdateSkill(userID, skillID int, req UpdateSkillRequest) return s.toSkillPayload(*item) } -func (s *skillService) DeleteSkill(userID, skillID int) error { +func (s *skillService) DeleteSkill(actorUserID int, actorRole string, skillID int) error { item, err := s.repo.GetSkillByID(skillID) if err != nil { return err } - if item == nil || item.UserID != userID { + if item == nil || isDeletedSkill(item) { + return fmt.Errorf("skill not found") + } + if !isAdminRole(actorRole) && item.UserID != actorUserID { return fmt.Errorf("skill not found") } if !isUserManagedSkill(*item) { return fmt.Errorf("skill not found") } - return s.repo.DeleteSkill(skillID) + now := time.Now().UTC() + item.Status = skillStatusDeleted + item.Visibility = skillVisibilityPrivate + item.PublishedAt = nil + item.PublishedBy = nil + item.SkillKey = deletedSkillKey(item.SkillKey, item.ID) + item.UpdatedAt = now + if err := s.repo.ReplaceSkillTagAssignments(skillID, []int{}); err != nil { + return err + } + return s.repo.UpdateSkill(item) } -func (s *skillService) DownloadSkill(userID, skillID int) ([]byte, string, error) { +func (s *skillService) DownloadSkill(actorUserID int, actorRole string, skillID int) ([]byte, string, error) { item, err := s.repo.GetSkillByID(skillID) if err != nil { return nil, "", err } - if item == nil || item.UserID != userID { - return nil, "", fmt.Errorf("skill not found") - } - if !isUserManagedSkill(*item) { + if item == nil || !s.CanDownloadSkill(actorUserID, actorRole, item) { return nil, "", fmt.Errorf("skill not found") } if item.CurrentVersionID == nil { @@ -345,10 +393,16 @@ func (s *skillService) DownloadSkill(userID, skillID int) ([]byte, string, error if err != nil { return nil, "", err } + if version == nil { + return nil, "", fmt.Errorf("skill version not found") + } blob, err := s.repo.GetBlobByID(version.BlobID) if err != nil { return nil, "", err } + if blob == nil || strings.TrimSpace(blob.ObjectKey) == "" { + return nil, "", fmt.Errorf("skill blob not found") + } content, err := s.storage.GetObject(context.Background(), blob.ObjectKey) if err != nil { return nil, "", err @@ -382,15 +436,12 @@ func (s *skillService) DownloadSkillVersionByExternalID(externalVersionID string return content, blob.FileName, nil } -func (s *skillService) ListVersions(userID, skillID int) ([]SkillVersionPayload, error) { +func (s *skillService) ListVersions(actorUserID int, actorRole string, skillID int) ([]SkillVersionPayload, error) { skill, err := s.repo.GetSkillByID(skillID) if err != nil { return nil, err } - if skill == nil || skill.UserID != userID { - return nil, fmt.Errorf("skill not found") - } - if !isUserManagedSkill(*skill) { + if skill == nil || !s.CanViewSkill(actorUserID, actorRole, skill) || !isUserManagedSkill(*skill) { return nil, fmt.Errorf("skill not found") } items, err := s.repo.ListVersionsBySkillID(skillID) @@ -416,6 +467,10 @@ func (s *skillService) ListInstanceSkills(instanceID int) ([]InstanceSkillPayloa if err := s.reconcileRemovedInstanceSkillsFromCommands(instanceID); err != nil { return nil, err } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, err + } items, err := s.repo.ListInstanceSkills(instanceID) if err != nil { return nil, err @@ -427,7 +482,7 @@ func (s *skillService) ListInstanceSkills(instanceID int) ([]InstanceSkillPayloa } payload := InstanceSkillPayload{ ID: item.ID, InstanceID: item.InstanceID, SkillID: item.SkillID, SkillVersionID: item.SkillVersionID, - SourceType: item.SourceType, InstallPath: item.InstallPath, ObservedHash: item.ObservedHash, + SourceType: item.SourceType, InstallPath: item.InstallPath, WorkspaceDir: item.WorkspaceDir, ObservedHash: item.ObservedHash, Status: item.Status, LastSeenAt: item.LastSeenAt, RemovedAt: item.RemovedAt, } skill, err := s.repo.GetSkillByID(item.SkillID) @@ -439,6 +494,9 @@ func (s *skillService) ListInstanceSkills(instanceID int) ([]InstanceSkillPayloa if err != nil { return nil, err } + if err := s.enrichSkillPayload(skillPayload, *skill, instance); err != nil { + return nil, err + } payload.Skill = skillPayload } result = append(result, payload) @@ -482,7 +540,7 @@ func commandFinishedAt(command InstanceCommandPayload) time.Time { return time.Time{} } -func (s *skillService) AttachSkillToInstanceForActor(instanceID int, skillID int, userID int, userRole string) (*InstanceSkillPayload, error) { +func (s *skillService) AttachSkillToInstance(actorUserID int, actorRole string, instanceID int, skillID int) (*InstanceSkillPayload, error) { instance, err := s.instanceRepo.GetByID(instanceID) if err != nil { return nil, err @@ -490,17 +548,9 @@ func (s *skillService) AttachSkillToInstanceForActor(instanceID int, skillID int if instance == nil { return nil, fmt.Errorf("instance not found") } - skill, err := s.repo.GetSkillByID(skillID) - if err != nil { + if err := EnsureInstanceWorkspacePathForServerScan(context.Background(), s.instanceRepo, instance); err != nil { return nil, err } - if skill == nil || !canActorAttachSkillToInstance(instance, *skill, userID, userRole) { - return nil, fmt.Errorf("skill not found") - } - return s.attachSkillModelToInstance(instanceID, skill) -} - -func (s *skillService) AttachSkillToInstance(instanceID int, skillID int) (*InstanceSkillPayload, error) { skill, err := s.repo.GetSkillByID(skillID) if err != nil { return nil, err @@ -508,10 +558,9 @@ func (s *skillService) AttachSkillToInstance(instanceID int, skillID int) (*Inst if skill == nil { return nil, fmt.Errorf("skill not found") } - return s.attachSkillModelToInstance(instanceID, skill) -} - -func (s *skillService) attachSkillModelToInstance(instanceID int, skill *models.Skill) (*InstanceSkillPayload, error) { + if !s.CanAttachSkill(actorUserID, actorRole, skill, instance) { + return nil, fmt.Errorf("skill_attach_forbidden") + } if !isUserManagedSkill(*skill) { return nil, fmt.Errorf("skill not found") } @@ -522,7 +571,6 @@ func (s *skillService) attachSkillModelToInstance(instanceID int, skill *models. return nil, fmt.Errorf("skill is blocked by risk policy") } - skillID := skill.ID versionID := skill.CurrentVersionID var blob *models.SkillBlob if versionID != nil { @@ -606,7 +654,7 @@ func (s *skillService) materializeLiteInstanceSkill(ctx context.Context, instanc return fmt.Errorf("lite skill materialization target is invalid") } - targetRoot := liteSkillInstallRoot(instance) + targetRoot := runtimeSkillInstallRoot(instance) if targetRoot == "" { return nil } @@ -616,15 +664,23 @@ func (s *skillService) materializeLiteInstanceSkill(ctx context.Context, instanc return ensureLiteRuntimePersistentOwnership(instance) } -func liteSkillInstallRoot(instance *models.Instance) string { - if instance == nil || instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" { - return "" +func resolveInstanceSkillSourceType(existing *models.InstanceSkill, incoming string, skill *models.Skill) string { + incoming = normalizeSkillSource(incoming) + if existing != nil && strings.EqualFold(strings.TrimSpace(existing.SourceType), "injected_by_clawmanager") { + return "injected_by_clawmanager" } - workspacePath := filepath.Clean(strings.TrimSpace(*instance.WorkspacePath)) - if strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeHermes) { - return filepath.Join(workspacePath, "home", ".hermes", "skills") + if incoming == "injected_by_clawmanager" { + return incoming } - return filepath.Join(workspacePath, "home", ".openclaw", "workspace", "skills") + if existing != nil && skill != nil && strings.EqualFold(strings.TrimSpace(skill.SourceType), skillSourceUploaded) { + if strings.EqualFold(strings.TrimSpace(existing.SourceType), "injected_by_clawmanager") { + return existing.SourceType + } + if incoming == "discovered_in_instance" && strings.TrimSpace(existing.SourceType) != "" { + return existing.SourceType + } + } + return incoming } func liteRuntimePersistentRoot(instance *models.Instance) string { @@ -712,21 +768,26 @@ func chownRuntimePath(targetPath string, uid, gid int, mode os.FileMode) error { return nil } -func writeSkillDirectoryAtomically(targetRoot, targetName string, files map[string][]byte) error { +func writeSkillDirectoryAtomically(targetRoot, relativePath string, files map[string][]byte) error { targetRoot = filepath.Clean(strings.TrimSpace(targetRoot)) - targetName = strings.TrimSpace(targetName) - if targetRoot == "." || targetRoot == "" || targetName == "" || strings.ContainsAny(targetName, `/\\`) { - return fmt.Errorf("invalid lite skill target") + relativePath = sanitizeWorkspaceRelativePath(relativePath) + if targetRoot == "." || targetRoot == "" || relativePath == "" { + return fmt.Errorf("invalid runtime skill target") + } + targetPath, err := joinRuntimeSkillPath(targetRoot, relativePath) + if err != nil { + return fmt.Errorf("invalid runtime skill target") } if err := os.MkdirAll(targetRoot, 0750); err != nil { - return fmt.Errorf("failed to prepare lite skill root: %w", err) + return fmt.Errorf("failed to prepare runtime skill root: %w", err) } tmpRoot := filepath.Join(targetRoot, ".tmp") if err := os.MkdirAll(tmpRoot, 0750); err != nil { - return fmt.Errorf("failed to prepare lite skill temp root: %w", err) + return fmt.Errorf("failed to prepare runtime skill temp root: %w", err) } - tmpDir, err := os.MkdirTemp(tmpRoot, ".tmp-skill-"+targetName+"-") + tmpNameSafe := strings.ReplaceAll(relativePath, "/", "-") + tmpDir, err := os.MkdirTemp(tmpRoot, ".tmp-skill-"+tmpNameSafe+"-") if err != nil { return fmt.Errorf("failed to create lite skill temp dir: %w", err) } @@ -758,9 +819,11 @@ func writeSkillDirectoryAtomically(targetRoot, targetName string, files map[stri } } - targetPath := filepath.Join(targetRoot, targetName) if !isPathWithin(targetRoot, targetPath) { - return fmt.Errorf("lite skill target escapes root") + return fmt.Errorf("runtime skill target escapes root") + } + if err := os.MkdirAll(filepath.Dir(targetPath), 0750); err != nil { + return fmt.Errorf("failed to prepare lite skill target parent: %w", err) } backupPath := targetPath + ".old" _ = os.RemoveAll(backupPath) @@ -797,7 +860,7 @@ func (s *skillService) RemoveSkillFromInstance(instanceID int, skillID int) erro if item == nil { return nil } - if err := s.removeLiteInstanceSkillDirectory(instanceID, item); err != nil { + if err := s.removeRuntimeInstanceSkillDirectory(instanceID, item); err != nil { return err } now := time.Now().UTC() @@ -818,7 +881,7 @@ func (s *skillService) RemoveSkillFromInstance(instanceID int, skillID int) erro return nil } -func (s *skillService) removeLiteInstanceSkillDirectory(instanceID int, item *models.InstanceSkill) error { +func (s *skillService) removeRuntimeInstanceSkillDirectory(instanceID int, item *models.InstanceSkill) error { if s == nil || s.instanceRepo == nil || item == nil { return nil } @@ -826,43 +889,61 @@ func (s *skillService) removeLiteInstanceSkillDirectory(instanceID int, item *mo if err != nil { return err } - if !isLiteRuntimeInstance(instance) { + if !isLiteRuntimeInstance(instance) && !SupportsServerWorkspaceSkillScan(instance) { return nil } - skillKey := strings.TrimSpace(skillKeyForRemoval(item)) - if skillKey == "" || strings.HasPrefix(skillKey, "skill-") { - skill, err := s.repo.GetSkillByID(item.SkillID) - if err != nil { - return err - } - if skill != nil { - skillKey = strings.TrimSpace(skill.SkillKey) - } + relativePath := "" + if item.WorkspaceDir != nil { + relativePath = sanitizeWorkspaceRelativePath(strings.TrimSpace(*item.WorkspaceDir)) } - targetName := sanitizeSkillKey(skillKey) - if targetName == "" { + if relativePath == "" { + skillKey := strings.TrimSpace(skillKeyForRemoval(item)) + if skillKey == "" || strings.HasPrefix(skillKey, "skill-") { + skill, err := s.repo.GetSkillByID(item.SkillID) + if err != nil { + return err + } + if skill != nil { + skillKey = strings.TrimSpace(skill.SkillKey) + } + } + relativePath = sanitizeWorkspaceRelativePath(skillKey) + } + if relativePath == "" { return nil } - targetRoot := liteSkillInstallRoot(instance) + targetRoot := runtimeSkillInstallRoot(instance) if targetRoot == "" { return nil } - targetPath := filepath.Join(targetRoot, targetName) - if !isPathWithin(targetRoot, targetPath) { - return fmt.Errorf("lite skill removal target escapes root") + targetPath, err := joinRuntimeSkillPath(targetRoot, relativePath) + if err != nil { + return fmt.Errorf("runtime skill removal target is invalid") } if err := os.RemoveAll(targetPath); err != nil { - return fmt.Errorf("failed to remove lite skill directory: %w", err) + return fmt.Errorf("failed to remove runtime skill directory: %w", err) } return ensureLiteRuntimePersistentOwnership(instance) } + +func (s *skillService) removeLiteInstanceSkillDirectory(instanceID int, item *models.InstanceSkill) error { + return s.removeRuntimeInstanceSkillDirectory(instanceID, item) +} func isBlockedSkillRisk(value string) bool { value = strings.TrimSpace(value) return strings.EqualFold(value, skillRiskMedium) || strings.EqualFold(value, skillRiskHigh) } func isRemovedInstanceSkill(item *models.InstanceSkill) bool { - return item != nil && (strings.EqualFold(strings.TrimSpace(item.Status), "removed") || item.RemovedAt != nil) + if item == nil { + return false + } + status := strings.ToLower(strings.TrimSpace(item.Status)) + return status == "removed" || status == "missing" +} + +func isUserRemovedInstanceSkill(item *models.InstanceSkill) bool { + return item != nil && strings.EqualFold(strings.TrimSpace(item.Status), "removed") } func (s *skillService) SyncAgentSkills(instanceID int, req AgentSkillInventoryReportRequest) error { @@ -881,13 +962,14 @@ func (s *skillService) SyncAgentSkills(instanceID int, req AgentSkillInventoryRe } active := make([]int, 0, len(req.Skills)) for _, record := range req.Skills { - hash := strings.TrimSpace(record.ContentMD5) + hash := workspaceContentHashForRecord(instance, record) if hash == "" { continue } normalizedSource := normalizeSkillSource(record.Source) var skill *models.Skill var version *models.SkillVersion + var blob *models.SkillBlob if normalizedSource == "injected_by_clawmanager" { if skillID, err := parseExternalSkillID(record.SkillID); err == nil { @@ -901,7 +983,10 @@ func (s *skillService) SyncAgentSkills(instanceID int, req AgentSkillInventoryRe } } - skillKey := sanitizeSkillKey(record.Identifier) + skillKey := skillKeyFromRelativePath(record.Identifier) + if skillKey == "" { + skillKey = sanitizeSkillKey(record.Identifier) + } if skillKey == "" { skillKey = hash[:skillMin(16, len(hash))] } @@ -910,49 +995,62 @@ func (s *skillService) SyncAgentSkills(instanceID int, req AgentSkillInventoryRe if err != nil { return err } - if item != nil && (normalizedSource != "discovered_in_instance" || strings.EqualFold(item.SourceType, skillSourceDiscovered)) { + if item != nil && (normalizedSource != "discovered_in_instance" || + strings.EqualFold(item.SourceType, skillSourceDiscovered) || + strings.EqualFold(item.SourceType, skillSourceUploaded)) { skill = item } } - blob, err := s.repo.GetBlobByContentHash(hash) - if err != nil { - return err + if skill != nil && liteInventoryUsesWorkspaceHash(instance) { + reconciledBlob, reconciledVersion, reconcileErr := s.reconcileLiteDiscoveredBlob(skill, hash) + if reconcileErr != nil { + return reconcileErr + } + if reconciledBlob != nil { + blob = reconciledBlob + version = reconciledVersion + } } - if blob == nil && skill != nil { - version, blob, err = s.findVersionByContentMD5(skill.ID, hash) + + if blob == nil { + var err error + blob, err = s.repo.GetBlobByContentHash(hash) if err != nil { return err } - } - if blob == nil { - blob = &models.SkillBlob{ - ContentHash: hash, - ArchiveHash: hash, - ObjectKey: "", - FileName: sanitizeSkillKey(record.Identifier) + ".zip", - MediaType: "application/zip", - SizeBytes: 0, - ScanStatus: "pending", - RiskLevel: skillRiskUnknown, + if blob == nil && skill != nil { + version, blob, err = s.findVersionByContentMD5(skill.ID, hash) + if err != nil { + return err + } } - if err := s.repo.CreateBlob(blob); err != nil { - return err + if blob == nil { + blob = &models.SkillBlob{ + ContentHash: hash, + ArchiveHash: hash, + ObjectKey: "", + FileName: sanitizeSkillKey(record.Identifier) + ".zip", + MediaType: "application/zip", + SizeBytes: 0, + ScanStatus: "pending", + RiskLevel: skillRiskUnknown, + } + if err := s.repo.CreateBlob(blob); err != nil { + return err + } } } if strings.TrimSpace(blob.ObjectKey) == "" { - _, _ = s.commandService.Create(instanceID, nil, CreateInstanceCommandRequest{ - CommandType: InstanceCommandTypeCollectSkillPackage, - Payload: map[string]interface{}{ + if !isLiteRuntimeInstance(instance) { + _ = s.enqueueCollectSkillPackage(instanceID, map[string]interface{}{ "skill_id": record.SkillID, "skill_version": record.SkillVersion, "identifier": record.Identifier, "content_md5": hash, "source": normalizedSource, - }, - IdempotencyKey: fmt.Sprintf("collect-skill-package-%d-%s", instanceID, hash), - TimeoutSeconds: 600, - }) + }, fmt.Sprintf("collect-skill-package-%d-%s", instanceID, hash)) + } } if skill == nil { if normalizedSource == "discovered_in_instance" { @@ -960,7 +1058,7 @@ func (s *skillService) SyncAgentSkills(instanceID int, req AgentSkillInventoryRe } skill = &models.Skill{ UserID: ownerUserID, SkillKey: skillKey, Name: strings.TrimSpace(record.Identifier), - SourceType: skillSourceDiscovered, Status: "active", RiskLevel: blob.RiskLevel, + SourceType: skillSourceDiscovered, Status: "active", Visibility: skillVisibilityPrivate, RiskLevel: blob.RiskLevel, LastScannedAt: blob.LastScannedAt, LastScanResultID: blob.LastScanResultID, } if skill.Name == "" { @@ -1003,21 +1101,40 @@ func (s *skillService) SyncAgentSkills(instanceID int, req AgentSkillInventoryRe if err != nil { return err } - if isRemovedInstanceSkill(existingInstanceSkill) { + if isUserRemovedInstanceSkill(existingInstanceSkill) { continue } active = append(active, skill.ID) + workspaceDir := sanitizeWorkspaceRelativePath(strings.TrimSpace(record.Identifier)) + resolvedSource := resolveInstanceSkillSourceType(existingInstanceSkill, normalizedSource, skill) instanceSkill := &models.InstanceSkill{ - InstanceID: instanceID, SkillID: skill.ID, SkillVersionID: optionalVersionID(version), SourceType: normalizedSource, + InstanceID: instanceID, SkillID: skill.ID, SkillVersionID: optionalVersionID(version), SourceType: resolvedSource, InstallPath: optionalString(strings.TrimSpace(record.InstallPath)), ObservedHash: optionalString(hash), - Status: "active", LastSeenAt: &reportedAt, UpdatedAt: reportedAt, + Status: "active", LastSeenAt: &reportedAt, UpdatedAt: reportedAt, RemovedAt: nil, + } + if workspaceDir != "" { + instanceSkill.WorkspaceDir = optionalString(workspaceDir) } if err := s.repo.UpsertInstanceSkill(instanceSkill); err != nil { return err } + if liteInventoryUsesWorkspaceHash(instance) && strings.TrimSpace(blob.ObjectKey) == "" && workspaceDir != "" && s.materializeService != nil { + if refreshedBlob, blobErr := s.repo.GetBlobByContentHash(hash); blobErr == nil && refreshedBlob != nil { + blob = refreshedBlob + } + _, _ = s.materializeService.Enqueue(context.Background(), EnqueueMaterializeRequest{ + InstanceID: instanceID, + SkillID: skill.ID, + BlobID: blob.ID, + WorkspaceDir: workspaceDir, + ContentHash: hash, + TriggerSource: MaterializeTriggerSync, + IdempotencyKey: fmt.Sprintf("materialize-%d-%s", instanceID, hash), + }) + } } - if strings.EqualFold(strings.TrimSpace(req.Mode), "full") || !strings.EqualFold(strings.TrimSpace(req.Mode), "incremental") { + if strings.EqualFold(strings.TrimSpace(req.Mode), "full") { if err := s.repo.MarkMissingInstanceSkills(instanceID, active, reportedAt); err != nil { return err } @@ -1057,55 +1174,19 @@ func (s *skillService) UploadAgentSkillPackage(ctx context.Context, instanceID i contentMD5 := hashDirectory(dir.Files) expectedMD5 := strings.TrimSpace(req.ContentMD5) if expectedMD5 != "" && !strings.EqualFold(contentMD5, expectedMD5) { - return nil, fmt.Errorf("skill package md5 mismatch: expected %s got %s", expectedMD5, contentMD5) + return nil, utils.NewHubError( + "skill_package_md5_mismatch", + fmt.Sprintf("skill package md5 mismatch: expected %s got %s", expectedMD5, contentMD5), + map[string]string{"expected": expectedMD5, "computed": contentMD5}, + ) } - archiveBytes, archiveHash, err := buildNormalizedZip(dir) + blob, err := s.persistDiscoveredSkillPackage(ctx, instanceID, dir, contentMD5, nil) if err != nil { return nil, err } - - blob, err := s.repo.GetBlobByContentHash(contentMD5) - if err != nil { - return nil, err - } - if blob == nil { - blob = &models.SkillBlob{ - ContentHash: contentMD5, - ArchiveHash: archiveHash, - ObjectKey: fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, sanitizeSkillKey(dir.Name), contentMD5), - FileName: fmt.Sprintf("%s.zip", sanitizeSkillKey(dir.Name)), - MediaType: "application/zip", - SizeBytes: int64(len(archiveBytes)), - ScanStatus: "pending", - RiskLevel: skillRiskUnknown, - } - if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil { - return nil, err - } - if err := s.repo.CreateBlob(blob); err != nil { - return nil, err - } - } else if strings.TrimSpace(blob.ObjectKey) == "" { - blob.ObjectKey = fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, sanitizeSkillKey(dir.Name), contentMD5) - blob.FileName = fmt.Sprintf("%s.zip", sanitizeSkillKey(dir.Name)) - blob.MediaType = "application/zip" - blob.SizeBytes = int64(len(archiveBytes)) - if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil { - return nil, err - } - if err := s.repo.UpdateBlob(blob); err != nil { - return nil, err - } - } - - if blob.LastScanResultID == nil || blob.ScanStatus != "completed" { - if err := s.recordScan(blob, &dir); err != nil { - blob.ScanStatus = "failed" - blob.UpdatedAt = time.Now().UTC() - _ = s.repo.UpdateBlob(blob) - return nil, err - } + if !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") { + return nil, fmt.Errorf("skill package scan failed") } normalizedSource := normalizeSkillSource(req.Source) @@ -1139,7 +1220,7 @@ func (s *skillService) UploadAgentSkillPackage(ctx context.Context, instanceID i if skill == nil { skill = &models.Skill{ UserID: instance.UserID, SkillKey: skillKey, Name: strings.TrimSpace(req.Identifier), - SourceType: skillSourceDiscovered, Status: "active", RiskLevel: blob.RiskLevel, + SourceType: skillSourceDiscovered, Status: "active", Visibility: skillVisibilityPrivate, RiskLevel: blob.RiskLevel, LastScannedAt: blob.LastScannedAt, LastScanResultID: blob.LastScanResultID, } if strings.TrimSpace(skill.Name) == "" { @@ -1196,12 +1277,12 @@ func (s *skillService) UploadAgentSkillPackage(ctx context.Context, instanceID i return s.toSkillPayload(*skill) } -func (s *skillService) ListScanResults(userID, skillID int) ([]SkillScanResultPayload, error) { +func (s *skillService) ListScanResults(actorUserID int, actorRole string, skillID int) ([]SkillScanResultPayload, error) { skill, err := s.repo.GetSkillByID(skillID) if err != nil { return nil, err } - if skill == nil || (skill.UserID != userID && userID != 0) { + if skill == nil || !s.CanViewSkill(actorUserID, actorRole, skill) { return nil, fmt.Errorf("skill not found") } if skill.CurrentVersionID == nil { @@ -1360,86 +1441,144 @@ func extractArchiveFileMap(filename string, raw []byte) (map[string][]byte, erro return fileMap, nil } -func (s *skillService) importDirectory(ctx context.Context, userID int, dir extractedSkillDirectory, originalName string) (*SkillPayload, error) { - skillKey := sanitizeSkillKey(dir.Name) - if skillKey == "" { - return nil, fmt.Errorf("skill directory name %q is invalid", dir.Name) +func (s *skillService) ensureBlobObject(ctx context.Context, blob *models.SkillBlob, archiveBytes []byte) error { + if strings.TrimSpace(blob.ObjectKey) == "" { + return fmt.Errorf("skill blob has no object key") } - contentHash := hashDirectory(dir.Files) - archiveBytes, archiveHash, err := buildNormalizedZip(dir) - if err != nil { - return nil, err + if _, err := s.storage.GetObject(ctx, blob.ObjectKey); err == nil { + return nil } + mediaType := strings.TrimSpace(blob.MediaType) + if mediaType == "" { + mediaType = "application/zip" + } + if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, mediaType); err != nil { + return fmt.Errorf("failed to restore skill blob object: %w", err) + } + blob.SizeBytes = int64(len(archiveBytes)) + blob.MediaType = mediaType + blob.UpdatedAt = time.Now().UTC() + return s.repo.UpdateBlob(blob) +} - blob, err := s.repo.GetBlobByContentHash(contentHash) - if err != nil { - return nil, err - } - if blob == nil { - blob = &models.SkillBlob{ - ContentHash: contentHash, ArchiveHash: archiveHash, - ObjectKey: fmt.Sprintf("%d/%s/%s.zip", userID, skillKey, contentHash), - FileName: fmt.Sprintf("%s.zip", skillKey), - MediaType: "application/zip", SizeBytes: int64(len(archiveBytes)), - ScanStatus: "pending", RiskLevel: skillRiskUnknown, - } - if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil { - return nil, err - } - if err := s.repo.CreateBlob(blob); err != nil { - return nil, err - } - if err := s.recordScan(blob, &dir); err != nil { - return nil, err - } - } +func (s *skillService) enqueueCollectSkillPackage(instanceID int, payload map[string]interface{}, idempotencyKey string) error { + _, err := s.commandService.Create(instanceID, nil, CreateInstanceCommandRequest{ + CommandType: InstanceCommandTypeCollectSkillPackage, + Payload: payload, + IdempotencyKey: idempotencyKey, + TimeoutSeconds: 600, + }) + return err +} - skill, err := s.repo.GetSkillByUserKey(userID, skillKey) +func (s *skillService) requestSkillPackageCollection(instanceID int, skill *models.Skill, instanceSkill *models.InstanceSkill, idempotencySuffix string) error { + blob, err := s.skillBlobForPublish(skill) if err != nil { - return nil, err + return err } - if skill == nil { - description := fmt.Sprintf("Imported from %s", originalName) - skill = &models.Skill{ - UserID: userID, SkillKey: skillKey, Name: dir.Name, Description: &description, - SourceType: skillSourceUploaded, Status: "active", RiskLevel: blob.RiskLevel, - LastScannedAt: blob.LastScannedAt, LastScanResultID: blob.LastScanResultID, - } - if err := s.repo.CreateSkill(skill); err != nil { - return nil, err - } + if strings.TrimSpace(blob.ObjectKey) != "" { + return nil } - version, err := s.repo.GetVersionBySkillAndBlob(skill.ID, blob.ID) + instance, err := s.instanceRepo.GetByID(instanceID) if err != nil { - return nil, err + return err + } + if instance != nil && isLiteRuntimeInstance(instance) { + if s.materializeService == nil { + return fmt.Errorf("skill_package_pending") + } + workspaceDir := resolveLiteWorkspaceDir(instanceSkill, skill) + if workspaceDir == "" { + return fmt.Errorf("skill_package_pending") + } + trigger := MaterializeTriggerRetry + if strings.HasPrefix(strings.TrimSpace(idempotencySuffix), "import-") { + trigger = MaterializeTriggerImport + } else if strings.HasPrefix(strings.TrimSpace(idempotencySuffix), "publish-") { + trigger = MaterializeTriggerPublish + } + if strings.TrimSpace(idempotencySuffix) == "" { + idempotencySuffix = fmt.Sprintf("%d-%d", instanceID, skill.ID) + } + job, err := s.materializeService.Enqueue(context.Background(), EnqueueMaterializeRequest{ + InstanceID: instanceID, + SkillID: skill.ID, + BlobID: blob.ID, + WorkspaceDir: workspaceDir, + ContentHash: s.resolveContentMD5(blob), + TriggerSource: trigger, + IdempotencyKey: fmt.Sprintf("materialize-%d-%s", instanceID, s.resolveContentMD5(blob)), + }) + if err != nil { + return err + } + if trigger != MaterializeTriggerSync && job != nil { + _ = s.materializeService.ProcessJob(context.Background(), job.ID) + latest, latestErr := s.materializeService.FindLatestBySkillID(skill.ID) + if latestErr == nil && latest != nil && strings.EqualFold(strings.TrimSpace(latest.Status), MaterializeJobStatusFailed) { + return fmt.Errorf("skill_package_materialize_failed") + } + } + refreshedBlob, blobErr := s.repo.GetBlobByID(blob.ID) + if blobErr == nil && refreshedBlob != nil && strings.TrimSpace(refreshedBlob.ObjectKey) != "" { + return nil + } + return fmt.Errorf("skill_package_pending") + } + payload := map[string]interface{}{ + "skill_id": formatExternalSkillID(skill.ID), + "identifier": skill.SkillKey, + "content_md5": s.resolveContentMD5(blob), + "source": instanceSkill.SourceType, + } + if skill.CurrentVersionID != nil { + payload["skill_version"] = formatExternalVersionID(*skill.CurrentVersionID) + } + if strings.TrimSpace(idempotencySuffix) == "" { + idempotencySuffix = fmt.Sprintf("%d-%d", instanceID, skill.ID) + } + _ = s.enqueueCollectSkillPackage(instanceID, payload, fmt.Sprintf("collect-skill-package-%s", idempotencySuffix)) + return fmt.Errorf("skill_package_pending") +} + +func (s *skillService) recordScanFromStoredBlob(blob *models.SkillBlob) error { + content, err := s.storage.GetObject(context.Background(), blob.ObjectKey) + if err != nil { + return fmt.Errorf("failed to read stored skill package: %w", err) + } + directories, err := extractSkillDirectories(blob.FileName, content) + if err != nil { + return err + } + if len(directories) == 0 { + return fmt.Errorf("no skill directories found in stored package") + } + return s.recordScan(blob, &directories[0]) +} + +func (s *skillService) promoteSkillToUploadedLibrary(skill *models.Skill) error { + if skill.CurrentVersionID == nil { + return fmt.Errorf("skill has no version") + } + version, err := s.repo.GetVersionByID(*skill.CurrentVersionID) + if err != nil { + return err } if version == nil { - latest, err := s.repo.GetLatestVersionBySkillID(skill.ID) - if err != nil { - return nil, err - } - versionNo := 1 - if latest != nil { - versionNo = latest.VersionNo + 1 - } - manifest, _ := json.Marshal(map[string]interface{}{"root_dir": dir.Name, "files": len(dir.Files)}) - manifestJSON := string(manifest) - version = &models.SkillVersion{ - SkillID: skill.ID, BlobID: blob.ID, VersionNo: versionNo, ManifestJSON: &manifestJSON, SourceType: skillSourceUploaded, - } - if err := s.repo.CreateVersion(version); err != nil { - return nil, err - } + return fmt.Errorf("skill has no version") } - skill.CurrentVersionID = &version.ID - skill.RiskLevel = blob.RiskLevel - skill.LastScannedAt = blob.LastScannedAt - skill.LastScanResultID = blob.LastScanResultID - skill.UpdatedAt = time.Now().UTC() + now := time.Now().UTC() + skill.SourceType = skillSourceUploaded + if strings.TrimSpace(skill.Visibility) == "" { + skill.Visibility = skillVisibilityPrivate + } + skill.UpdatedAt = now + version.SourceType = skillSourceUploaded + version.UpdatedAt = now if err := s.repo.UpdateSkill(skill); err != nil { - return nil, err + return err } - return s.toSkillPayload(*skill) + return s.repo.UpdateVersion(version) } func (s *skillService) recordScan(blob *models.SkillBlob, dir *extractedSkillDirectory) error { @@ -1549,6 +1688,9 @@ func (s *skillService) resolveContentMD5(blob *models.SkillBlob) string { if len(contentHash) == 32 { return contentHash } + if s.storage == nil || strings.TrimSpace(blob.ObjectKey) == "" { + return contentHash + } content, err := s.storage.GetObject(context.Background(), blob.ObjectKey) if err != nil { return contentHash @@ -1645,13 +1787,18 @@ func sanitizeSkillKey(value string) string { return result } -func (s *skillService) toSkillPayloads(items []models.Skill) ([]SkillPayload, error) { +func (s *skillService) toSkillPayloads(items []models.Skill, actorUserID int, actorRole string) ([]SkillPayload, error) { result := make([]SkillPayload, 0, len(items)) for _, item := range items { payload, err := s.toSkillPayload(item) if err != nil { return nil, err } + if actorUserID > 0 || isAdminRole(actorRole) { + if err := s.enrichSkillPayload(payload, item, nil); err != nil { + return nil, err + } + } result = append(result, *payload) } return result, nil @@ -1661,8 +1808,12 @@ func (s *skillService) toSkillPayload(item models.Skill) (*SkillPayload, error) payload := &SkillPayload{ ID: item.ID, ExternalSkillID: formatExternalSkillID(item.ID), UserID: item.UserID, SkillKey: item.SkillKey, Name: item.Name, Description: item.Description, Status: item.Status, SourceType: item.SourceType, RiskLevel: item.RiskLevel, ScanStatus: "pending", + Visibility: skillVisibilityPrivate, LastScannedAt: item.LastScannedAt, CurrentVersionID: item.CurrentVersionID, CreatedAt: item.CreatedAt, UpdatedAt: item.UpdatedAt, } + if strings.TrimSpace(item.Visibility) != "" { + payload.Visibility = item.Visibility + } if item.CurrentVersionID != nil { version, err := s.repo.GetVersionByID(*item.CurrentVersionID) if err != nil { @@ -1789,31 +1940,14 @@ func severityRank(value string) int { } func (s *skillService) findInstanceRefs(skillID int) (int, error) { - all, err := s.repo.ListAllSkills() + if s.repo == nil { + return 0, nil + } + items, err := s.repo.ListActiveInstanceSkillsBySkillID(skillID) if err != nil { return 0, err } - _ = all - count := 0 - for instanceID := 1; instanceID <= 0; instanceID++ { - _ = instanceID - } - instances, err := s.instanceRepo.GetAll(0, 100000) - if err != nil { - return 0, err - } - for _, instance := range instances { - items, err := s.repo.ListInstanceSkills(instance.ID) - if err != nil { - return 0, err - } - for _, item := range items { - if item.SkillID == skillID && item.Status != "removed" { - count++ - } - } - } - return count, nil + return len(items), nil } func normalizeSkillSource(value string) string { @@ -1838,6 +1972,34 @@ func isUserManagedSkill(skill models.Skill) bool { return strings.EqualFold(strings.TrimSpace(skill.SourceType), skillSourceUploaded) } +func isDeletedSkill(skill *models.Skill) bool { + if skill == nil { + return false + } + return strings.EqualFold(strings.TrimSpace(skill.Status), skillStatusDeleted) +} + +func deletedSkillKey(skillKey string, skillID int) string { + const maxSkillKeyLength = 120 + suffix := fmt.Sprintf("__deleted_%d", skillID) + trimmed := strings.TrimSpace(skillKey) + if trimmed == "" { + trimmed = "skill" + } + if strings.HasSuffix(trimmed, suffix) { + return trimmed + } + if len(trimmed)+len(suffix) <= maxSkillKeyLength { + return trimmed + suffix + } + runes := []rune(trimmed) + maxPrefixLength := maxSkillKeyLength - len(suffix) + if len(runes) > maxPrefixLength { + runes = runes[:maxPrefixLength] + } + return string(runes) + suffix +} + func optionalVersionID(version *models.SkillVersion) *int { if version == nil { return nil diff --git a/backend/internal/services/skill_service_test.go b/backend/internal/services/skill_service_test.go index 91db8a8..feb4652 100644 --- a/backend/internal/services/skill_service_test.go +++ b/backend/internal/services/skill_service_test.go @@ -294,6 +294,19 @@ func TestChownRuntimePathReportsRootPermissionDenied(t *testing.T) { t.Fatalf("chownRuntimePath() error = %v, want owner error", err) } } +func TestWriteSkillDirectoryAtomicallyNestedCategoryPath(t *testing.T) { + targetRoot := t.TempDir() + err := writeSkillDirectoryAtomically(targetRoot, "productivity/my-skill", map[string][]byte{ + "SKILL.md": []byte("# Nested Skill\n"), + }) + if err != nil { + t.Fatalf("writeSkillDirectoryAtomically() error = %v", err) + } + target := filepath.Join(targetRoot, "productivity", "my-skill", "SKILL.md") + if _, err := os.Stat(target); err != nil { + t.Fatalf("expected nested skill directory, stat err = %v", err) + } +} func TestWriteSkillDirectoryAtomicallyUsesNestedTempRoot(t *testing.T) { targetRoot := t.TempDir() err := writeSkillDirectoryAtomically(targetRoot, "marker-pdf-ingest", map[string][]byte{ diff --git a/backend/internal/utils/hub_errors.go b/backend/internal/utils/hub_errors.go new file mode 100644 index 0000000..9a558b6 --- /dev/null +++ b/backend/internal/utils/hub_errors.go @@ -0,0 +1,54 @@ +package utils + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" +) + +type HubError struct { + Code string + Message string + Details map[string]string +} + +func (e *HubError) Error() string { + if e == nil { + return "" + } + if e.Message != "" { + return e.Message + } + return e.Code +} + +func NewHubError(code, message string, details map[string]string) *HubError { + return &HubError{Code: code, Message: message, Details: details} +} + +// HandleHubError maps skill hub domain errors to HTTP responses. +func HandleHubError(c *gin.Context, err error) { + var hubErr *HubError + if errors.As(err, &hubErr) { + switch hubErr.Code { + case "skill_package_md5_mismatch": + Error(c, http.StatusBadRequest, hubErr.Code) + default: + Error(c, http.StatusBadRequest, hubErr.Error()) + } + return + } + switch err.Error() { + case "skill_not_scanned", "skill_risk_blocked", "skill_tags_required", "skill_not_in_library", "skill is not published to hub": + Error(c, http.StatusBadRequest, err.Error()) + case "skill_package_pending": + Error(c, http.StatusConflict, err.Error()) + case "skill_package_materialize_failed", "skill_package_materializing": + Error(c, http.StatusConflict, err.Error()) + case "skill_attach_forbidden", "access denied": + Error(c, http.StatusForbidden, err.Error()) + default: + HandleError(c, err) + } +} diff --git a/backend/internal/utils/response.go b/backend/internal/utils/response.go index 8ba0b9d..aaca9bc 100644 --- a/backend/internal/utils/response.go +++ b/backend/internal/utils/response.go @@ -72,12 +72,16 @@ func HandleError(c *gin.Context, err error) { Error(c, http.StatusForbidden, errStr) case "invalid username or password", "account is disabled", "invalid or expired agent session token": Error(c, http.StatusUnauthorized, errStr) - case "agent registration is only supported for openclaw instances", "agent registration is only supported for openclaw or hermes instances", "agent id does not match session", "access denied": + case "agent registration is only supported for openclaw instances", "agent registration is only supported for openclaw or hermes instances", "agent id does not match session", "access denied", "skill_attach_forbidden": Error(c, http.StatusForbidden, errStr) case "current password is incorrect": Error(c, http.StatusBadRequest, errStr) - case "user not found", "model not found": + case "user not found", "model not found", "skill not found", "skill hub tag not found": Error(c, http.StatusNotFound, errStr) + case "skill_not_scanned", "skill_risk_blocked", "skill_tags_required", "skill is not published to hub": + Error(c, http.StatusBadRequest, errStr) + case "skill_package_pending", "skill_package_materialize_failed", "skill_package_materializing": + Error(c, http.StatusConflict, err.Error()) default: // For development, show actual error; for production, hide details Error(c, http.StatusInternalServerError, errStr) diff --git a/backend/internal/utils/session_format.go b/backend/internal/utils/session_format.go new file mode 100644 index 0000000..a14b20c --- /dev/null +++ b/backend/internal/utils/session_format.go @@ -0,0 +1,43 @@ +package utils + +import "strings" + +// FormatOpenClawSessionKey extracts the display session key from a stored session ID. +func FormatOpenClawSessionKey(sessionID string) string { + sessionID = strings.TrimSpace(sessionID) + for _, prefix := range []string{"agent:openclaw:", "agent:hermes:"} { + if strings.HasPrefix(sessionID, prefix) { + return strings.TrimPrefix(sessionID, prefix) + } + } + if strings.HasPrefix(sessionID, "agent:") { + parts := strings.SplitN(sessionID, ":", 3) + if len(parts) == 3 && strings.TrimSpace(parts[2]) != "" { + return parts[2] + } + } + return sessionID +} + +// NormalizeOpenClawSessionID maps a runtime session key to the canonical stored session ID. +func NormalizeOpenClawSessionID(sessionKey string, runtimeType string) string { + sessionKey = strings.TrimSpace(sessionKey) + if sessionKey == "" { + return sessionKey + } + if strings.HasPrefix(sessionKey, "agent:") { + return sessionKey + } + switch strings.ToLower(strings.TrimSpace(runtimeType)) { + case "hermes": + return "agent:hermes:" + sessionKey + default: + return "agent:openclaw:" + sessionKey + } +} + +// IsTraceFallbackSessionID reports whether a session ID was generated per trace. +func IsTraceFallbackSessionID(sessionID string) bool { + sessionID = strings.TrimSpace(sessionID) + return strings.HasPrefix(sessionID, "sess_") +} diff --git a/backend/internal/utils/session_format_test.go b/backend/internal/utils/session_format_test.go new file mode 100644 index 0000000..60d2490 --- /dev/null +++ b/backend/internal/utils/session_format_test.go @@ -0,0 +1,36 @@ +package utils + +import "testing" + +func TestFormatOpenClawSessionKey(t *testing.T) { + if got := FormatOpenClawSessionKey("agent:openclaw:main"); got != "main" { + t.Fatalf("expected main, got %q", got) + } + if got := FormatOpenClawSessionKey("agent:hermes:work"); got != "work" { + t.Fatalf("expected work, got %q", got) + } + if got := FormatOpenClawSessionKey("sess_trc_123"); got != "sess_trc_123" { + t.Fatalf("expected passthrough, got %q", got) + } +} + +func TestNormalizeOpenClawSessionID(t *testing.T) { + if got := NormalizeOpenClawSessionID("main", "openclaw"); got != "agent:openclaw:main" { + t.Fatalf("expected agent:openclaw:main, got %q", got) + } + if got := NormalizeOpenClawSessionID("main", "hermes"); got != "agent:hermes:main" { + t.Fatalf("expected agent:hermes:main, got %q", got) + } + if got := NormalizeOpenClawSessionID("agent:openclaw:main", "openclaw"); got != "agent:openclaw:main" { + t.Fatalf("expected unchanged, got %q", got) + } +} + +func TestIsTraceFallbackSessionID(t *testing.T) { + if !IsTraceFallbackSessionID("sess_trc_abc") { + t.Fatal("expected trace fallback session") + } + if IsTraceFallbackSessionID("agent:openclaw:main") { + t.Fatal("expected stable session") + } +} diff --git a/deployments/k8s/cluster/instance-egress-networkpolicy.yaml b/deployments/k8s/cluster/instance-egress-networkpolicy.yaml new file mode 100644 index 0000000..114775e --- /dev/null +++ b/deployments/k8s/cluster/instance-egress-networkpolicy.yaml @@ -0,0 +1,40 @@ +# Optional cluster-wide reference policy for managed OpenClaw/Hermes runtimes. +# Enable per-instance enforcement by setting CLAWMANAGER_INSTANCE_NETWORK_LOCK=true +# on the ClawManager backend deployment. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: clawmanager-managed-runtime-egress + namespace: clawmanager-system + labels: + app: clawmanager + policy-role: managed-runtime-egress-reference +spec: + podSelector: + matchLabels: + clawmanager.io/managed-runtime: "true" + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: clawmanager-system + ports: + - protocol: TCP + port: 80 + - protocol: TCP + port: 443 + - protocol: TCP + port: 9001 + - protocol: TCP + port: 3128 diff --git a/deployments/k8s/single-node/clawmanager.yaml b/deployments/k8s/single-node/clawmanager.yaml index b0652d7..08463d8 100644 --- a/deployments/k8s/single-node/clawmanager.yaml +++ b/deployments/k8s/single-node/clawmanager.yaml @@ -22,9 +22,9 @@ stringData: runtime-agent-report-token: change-me-runtime-report-token openclaw-gateway-token: change-me-openclaw-gateway-token minio-root-user: minioadmin - minio-root-password: minioadmin123 + minio-root-password: minioadmin@123 minio-access-key: minioadmin - minio-secret-key: minioadmin123 + minio-secret-key: minioadmin@123 --- apiVersion: v1 kind: ConfigMap @@ -906,6 +906,7 @@ spec: spec: containers: - name: mysql + #image: mysql:8.4.8 image: mysql:8.4.8 imagePullPolicy: IfNotPresent args: @@ -1158,6 +1159,7 @@ subjects: name: clawmanager-app namespace: clawmanager-system --- +--- # Explicit lease permissions for control-plane leader election. The # cluster-admin binding above already covers this; this Role documents the # requirement and keeps leader election working if RBAC is tightened later. diff --git a/deployments/k8s/single-node/instance-egress-networkpolicy.yaml b/deployments/k8s/single-node/instance-egress-networkpolicy.yaml new file mode 100644 index 0000000..b312766 --- /dev/null +++ b/deployments/k8s/single-node/instance-egress-networkpolicy.yaml @@ -0,0 +1,43 @@ +# Optional cluster-wide reference policy for managed OpenClaw/Hermes runtimes. +# Enable per-instance enforcement by setting CLAWMANAGER_INSTANCE_NETWORK_LOCK=true +# on the ClawManager backend deployment. +# +# Per-instance policies are created automatically for managed runtimes when the +# lock is enabled. This manifest documents the intended egress shape. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: clawmanager-managed-runtime-egress + namespace: clawmanager-system + labels: + app: clawmanager + policy-role: managed-runtime-egress-reference +spec: + podSelector: + matchLabels: + clawmanager.io/managed-runtime: "true" + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: clawmanager-system + ports: + - protocol: TCP + port: 80 + - protocol: TCP + port: 443 + - protocol: TCP + port: 9001 + - protocol: TCP + port: 3128 diff --git a/docs/runtime-agent-integration-guide.md b/docs/runtime-agent-integration-guide.md index 13c0053..53b2d2e 100644 --- a/docs/runtime-agent-integration-guide.md +++ b/docs/runtime-agent-integration-guide.md @@ -93,6 +93,44 @@ Agent 启动时如果 `CLAWMANAGER_AGENT_ENABLED` 不是 `true`，应进入空 Runtime 内的应用和 agent 如果需要调用模型，优先使用这些变量，不要让用户在镜像内手工写入 provider key。 +### LLM Session 归因 + +托管 runtime 的每次 LLM 请求应携带稳定 session 标识，供平台按会话统计 token： + +| 项 | 要求 | +| --- | --- | +| Header | `x-openclaw-session-key: {sessionKey}` | +| 示例 | `main` 会归一化为 `agent:openclaw:main` | +| 托管实例 Gateway Token 默认 | 未显式传 key 时，ClawManager 默认使用 `main` | +| 备选 | 请求体 `session_id` 或 OpenAI `user` 字段 | +| 禁止 | 长期依赖 Gateway 自动生成的 `sess_{traceID}`（用户 JWT 等非托管调用仍会 fallback） | + +Agent state report 可选上报 LLM 配置指纹，便于平台检测配置漂移： + +```json +{ + "runtime": { + "llm_config_status": "gateway", + "llm_provider_base_url": "http://clawmanager-gateway.../api/v1/gateway/llm", + "llm_config_fingerprint": "sha256..." + } +} +``` + +### Egress 代理与实例归因 + +托管 runtime 实例会注入 egress 代理环境变量，并将实例 ID 写入 `CLAWMANAGER_EGRESS_INSTANCE_ID`。当 egress 拦截直连 LLM 提供商域名时，平台会把该事件记为 `egress.llm.blocked` 审计。 + +| 变量 / Header | 说明 | +| --- | --- | +| `HTTP_PROXY` / `HTTPS_PROXY` | 指向 ClawManager egress proxy | +| `CLAWMANAGER_EGRESS_INSTANCE_ID` | 当前实例 ID，供代理客户端上报 | +| `X-ClawManager-Instance-Id` | egress 请求应携带的实例 ID header（与 `X-ClawManager-Egress-Instance-Id` 等价） | + +建议：任何从实例内主动发起的 egress CONNECT/HTTP 代理请求（包括自定义脚本、sidecar、调试工具）在可行时读取 `CLAWMANAGER_EGRESS_INSTANCE_ID` 并设置 `X-ClawManager-Instance-Id`，以便平台将 bypass 尝试关联到具体实例。 + +可选网络加固：在 ClawManager 后端设置 `CLAWMANAGER_INSTANCE_NETWORK_LOCK=true` 后，新建的 **Pro（独立 Pod）** OpenClaw/Hermes 实例会自动创建 egress NetworkPolicy。**Lite（gateway 池）** 实例共享 runtime Pod，不适用按实例 NetworkPolicy。参考 `deployments/k8s/single-node/instance-egress-networkpolicy.yaml`。 + ## Agent 生命周期 推荐主循环： @@ -342,7 +380,7 @@ Authorization: Bearer {session_token} | `collect_system_info` | 立即采样，发送 state report，并在 finish result 中带上同一份摘要 | | `health_check` | 检查主进程、桌面入口、agent、metrics collector，并发送 state report | | `sync_skill_inventory` | 扫描 skill 目录并上报完整 inventory | -| `refresh_skill_inventory` | 重新扫描 skill 目录并上报完整 inventory | +| `refresh_skill_inventory` | **已废弃**。请使用 `sync_skill_inventory` | | `collect_skill_package` | 打包指定 skill 并上传 | | `install_skill` | 下载并安装平台指定 skill version | | `update_skill` | 更新已安装 skill | diff --git a/docs/session-token-usage.md b/docs/session-token-usage.md new file mode 100644 index 0000000..4da4e64 --- /dev/null +++ b/docs/session-token-usage.md @@ -0,0 +1,176 @@ +# Session Token Usage + +Instance-level reporting for LLM calls routed through the platform AI Gateway. + +## Data flow + +1. Runtime or user calls `POST /api/v1/gateway/llm/chat/completions`. +2. AI Gateway persists: + - `model_invocations` (tokens, model, status, `instance_id`, `session_id`) + - `cost_records` (estimated cost per trace) + - `audit_events` (including `gateway.session.fallback` when session key is missing) + - `chat_sessions` (optional title from first user message) +3. Instance detail UI calls: + - `GET /api/v1/instances/:id/session-usage` + - `GET /api/v1/instances/:id/session-usage/detail?session_id=...` +4. Admin overview UI calls: + - `GET /api/v1/admin/session-usage/overview` + +## Session ID rules + +| Source | Stored `session_id` | +|--------|---------------------| +| Header `x-openclaw-session-key: main` on OpenClaw | `agent:openclaw:main` | +| Header on Hermes | `agent:hermes:{key}` | +| Missing stable key (user JWT or non-managed callers) | `sess_trc_{traceId}` (fallback) | +| Missing key on **instance gateway token** for OpenClaw/Hermes | `agent:{type}:main` (managed default) | + +Display keys are derived via `FormatOpenClawSessionKey` (e.g. `main`). + +## API + +### List session usage + +`GET /api/v1/instances/:id/session-usage?page=1&limit=20&search=main&since=2026-07-01T00:00:00Z` + +Query parameters: + +- `since` / `until`: optional RFC3339 timestamps (`until` must be after `since`); filter on invocation `created_at` (cost aggregates join non-blocked invocations on the same window) +- `search`: filters session rows by session id/key/title (summary totals ignore search) + +Response highlights: + +- `summary`: totals across all sessions on the instance +- `compliance`: fallback session count and recent fallback audit events +- `items`: paginated per-session rows + +### Session detail + +`GET /api/v1/instances/:id/session-usage/detail?session_id=agent:openclaw:main&since=2026-07-01T00:00:00Z` + +Accepts the same optional `since` / `until` bounds as the list endpoint. Detail rows, model breakdown, and recent traces respect the time window and exclude blocked invocations. + +Returns model breakdown (tokens + cost) and recent traces for one session. + +### Admin cross-instance overview + +`GET /api/v1/admin/session-usage/overview?page=1&limit=20&search=openclaw&since=2026-07-01T00:00:00Z` + +Returns managed OpenClaw/Hermes running instances sorted by total tokens, with per-instance summary and global totals. + +## UI features + +- **Time range presets**: all time, 24h, 7d, 30d (instance panel and admin overview) +- **Auto refresh**: optional 15s polling +- **CSV export**: instance panel exports all filtered session rows; admin page exports instance summary rows + +## Limits + +- **Gateway only**: direct external LLM calls that bypass the platform gateway are not included. +- **Blocked invocations** are excluded from token aggregates. +- Supported instance types in UI: `openclaw`, `hermes`. + +## Database indexes + +Migration `038_add_session_usage_indexes.sql` adds: + +- `cost_records(instance_id)` +- `cost_records(session_id)` +- `model_invocations(instance_id, session_id, created_at)` + +## Local verification + +1. Apply migrations (including `038`). +2. Open an OpenClaw or Hermes instance detail page (Lite or Pro). +3. Send a gateway chat completion with `x-openclaw-session-key: main`. +4. Refresh the **Session Token Usage** panel and confirm token totals increase. +5. Open **Admin → AI Gateway → Session Usage** for the cross-instance overview. + +Optional E2E: + +```bash +cd e2e +npx playwright test tests/instances/session-token-tracking.spec.ts +``` + +Requires a running stack, configured gateway models, and DB access for `fixtures/dbClient.ts`. + +## E2E coverage + +| Spec | Scope | +|------|-------| +| `session-token-tracking.spec.ts` | Instance session usage API, gateway aggregation, fallback compliance, instance gateway token | +| `session-usage-admin.spec.ts` | Admin overview API, `since` query validation, non-admin 403 | + +Run all session usage specs: + +```bash +cd e2e +npx playwright test tests/instances/session-token-tracking.spec.ts tests/instances/session-usage-admin.spec.ts +``` + +## Pre-commit checklist + +When preparing the standalone session-usage PR: + +1. Branch: `feat/session-token-usage` (from Skill Hub baseline) +2. Include migration `038_add_session_usage_indexes.sql` +3. Exclude unrelated WIP: egress policy, local deployment yaml, debug `_*.json` artifacts +4. Suggested commit split: + - `feat(session-usage): add session usage APIs, indexes, and admin overview` + - `feat(session-usage): add instance/admin UI with filters, refresh, and CSV export` + - `test(session-usage): add handler, service, repository, and e2e coverage` + - `docs(session-usage): add session token usage guide` +5. Verify locally: + - `go test ./internal/services/... ./internal/handlers/... ./internal/repository/... -run SessionUsage` + - Playwright P1 specs above (P0 gateway aggregation may skip when upstream LLM unavailable) + +## Phase 9 staging file list + +Include (session usage only): + +**Backend** +- `backend/cmd/server/main.go` (session-usage routes only — review diff before staging) +- `backend/internal/db/migrations/038_add_session_usage_indexes.sql` +- `backend/internal/db/migrations_test.go` (038 test) +- `backend/internal/handlers/session_usage_query.go` +- `backend/internal/handlers/session_usage_query_test.go` +- `backend/internal/handlers/instance_handler.go` +- `backend/internal/handlers/instance_handler_test.go` +- `backend/internal/handlers/ai_observability_handler.go` +- `backend/internal/handlers/ai_observability_handler_test.go` +- `backend/internal/repository/session_usage_filter.go` +- `backend/internal/repository/session_usage_filter_test.go` +- `backend/internal/repository/model_invocation_repository.go` +- `backend/internal/repository/cost_record_repository.go` +- `backend/internal/services/ai_observability_service.go` +- `backend/internal/services/ai_observability_session_usage_test.go` + +**Frontend** +- `frontend/src/components/InstanceSessionUsagePanel.tsx` +- `frontend/src/pages/admin/SessionUsageOverviewPage.tsx` +- `frontend/src/pages/instances/InstanceDetailPage.tsx` +- `frontend/src/components/AdminLayout.tsx` +- `frontend/src/pages/admin/AIGatewayPage.tsx` +- `frontend/src/router/index.tsx` +- `frontend/src/services/instanceService.ts` +- `frontend/src/services/adminService.ts` +- `frontend/src/types/instance.ts` +- `frontend/src/utils/sessionUsageExport.ts` +- `frontend/src/lib/i18n.ts` + +**E2E & docs** +- `e2e/fixtures/apiClient.ts` +- `e2e/fixtures/dbClient.ts` +- `e2e/tests/instances/session-token-tracking.spec.ts` +- `e2e/tests/instances/session-usage-admin.spec.ts` +- `docs/session-token-usage.md` + +Exclude (do not stage for session-usage PR): + +- `backend/internal/egresspolicy/**` +- `backend/internal/handlers/egress_proxy_handler*.go` +- `deployments/k8s/**/instance-egress-networkpolicy.yaml` +- `deployments/scripts/**` +- `e2e/tests/instances/llm-governance.spec.ts` (unless bundled intentionally) +- Root `_*.json`, `pr138.patch`, debug artifacts diff --git a/docs/skill-content-md5-spec.md b/docs/skill-content-md5-spec.md index 82573aa..f493b4a 100644 --- a/docs/skill-content-md5-spec.md +++ b/docs/skill-content-md5-spec.md @@ -132,3 +132,13 @@ weather/ Hermes agent 在本地计算 MD5 时应对目录 `/config/.hermes/skills/weather` 调用 `skill_content_md5()`。不要对 zip 文件调用 MD5。 如果 agent 需要在上传前自检，可以先把 zip 解开，确认去掉 `weather/` 后得到的文件列表与本地计算使用的相对路径一致。 + +## 常见 MD5 不匹配排查 + +| 现象 | 可能原因 | 处理 | +|------|----------|------| +| `expected abc got def` 且 expected 来自 inventory | Agent inventory 与 collect 使用了不同目录快照 | 确保 collect 前目录未变化，且两次调用同一 `skill_content_md5()` | +| got 值每次不同 | 对 zip 文件 bytes 做 MD5，或 zip 内文件顺序/元数据参与计算 | 只对 skill 目录内容做规范化 MD5，见上文算法 | +| expected 含顶层目录名 | inventory 多剥了一层 skill 根目录 | 基准目录应为 `/config/.openclaw/workspace/skills/{name}/` 内部 | +| 隐藏文件导致偏差 | `.git`、`.cache` 等被计入或遗漏不一致 | 跳过任意以 `.` 开头的路径段 | +| collect 命令 succeeded 但 Hub 仍 unknown | skill-scanner 未部署或扫描失败 | 检查 `skill-scanner` Pod 与 blob `scan_status` | diff --git a/docs/skill-lite-materialize.md b/docs/skill-lite-materialize.md new file mode 100644 index 0000000..5f44eb5 --- /dev/null +++ b/docs/skill-lite-materialize.md @@ -0,0 +1,57 @@ +# Lite Skill Package Materialization + +Lite (gateway / Hermes) instances discover skills from the shared workspace instead of +using the instance agent `collect_skill_package` command. + +## Lifecycle + +1. **Inventory** — `syncLiteSkillsFromWorkspace` or runtime agent report calls + `SyncAgentSkills`, which upserts skills and writes `instance_skills.workspace_dir`. +2. **Enqueue** — For Lite instances with empty `skill_blobs.object_key`, ClawManager + inserts a row into `skill_package_materialize_jobs` (never `collect_skill_package`). +3. **Materialize** — The leader-only `SkillPackageMaterializeWorker` reads workspace + directories, builds a normalized ZIP, uploads to MinIO, and runs skill-scanner. +4. **Publish** — Once `object_key` is set and scan completes, skills can be imported + to the library and published to Skill Hub. + +## Paths + +| Runtime | Workspace skill root | +|---------|---------------------| +| Hermes Lite | `{workspace}/home/.hermes/skills/{name}` | +| OpenClaw Lite | `{workspace}/home/.openclaw/workspace/skills/{name}` | + +The authoritative directory name is stored in `instance_skills.workspace_dir`. + +## Configuration + +| Environment variable | Default | Description | +|------------------------|---------|-------------| +| `SKILL_MATERIALIZE_WORKER_ENABLED` | `true` | Enable background worker | +| `SKILL_MATERIALIZE_TICK_MS` | `2000` | Worker poll interval | +| `SKILL_MATERIALIZE_BATCH_SIZE` | `5` | Jobs claimed per tick | +| `SKILL_MATERIALIZE_CONCURRENCY` | `5` | Global worker concurrency | +| `SKILL_MATERIALIZE_PER_INSTANCE_CONCURRENCY` | `2` | Max parallel jobs per instance | + +## Agent commands + +Pro and Shell instances still use `collect_skill_package` via the instance agent. +Lite instances **do not**; package collection is server-side only. + +For Lite inventory, ClawManager treats the shared workspace scan as the authoritative +`content_md5` source. Runtime agent reports may differ; server-side materialize always +recomputes from workspace and self-heals stale blob hashes instead of failing with +`skill package md5 mismatch`. + +## Backfill + +On worker start, pending Lite blobs with `workspace_dir` set are enqueued automatically. +Migration `039_add_skill_package_materialize.sql` also cancels stale Lite +`collect_skill_package` commands and backfills `workspace_dir` from `install_path`. + +## Hub UI blocked reasons + +When enriching Skill Hub payloads without an explicit instance (catalog, "My Skills", +detail pages), ClawManager resolves a Lite instance from active `instance_skills` rows +for that skill. This prevents stale Pro-only `collect_skill_package` failures from +showing as `skill_package_collect_failed` on Lite-discovered skills. diff --git a/e2e/fixtures/apiClient.ts b/e2e/fixtures/apiClient.ts index f2e11cc..d44ee37 100644 --- a/e2e/fixtures/apiClient.ts +++ b/e2e/fixtures/apiClient.ts @@ -396,3 +396,139 @@ export async function disableExternalAccess( }); await expectOkEnvelope(response); } + +export interface GatewayModelSummary { + id: number | string; + display_name?: string; +} + +export async function listGatewayModels( + request: APIRequestContext, + accessToken: string, +): Promise { + const response = await request.get(`${env.backendUrl}/gateway/llm/models`, { + headers: bearer(accessToken), + }); + const body = await expectOkEnvelope<{ items: GatewayModelSummary[] }>(response); + return body.items ?? []; +} + +export async function gatewayChatCompletion( + request: APIRequestContext, + accessToken: string, + payload: { + model: string; + instance_id?: number; + messages: Array<{ role: string; content: string }>; + stream?: boolean; + }, + extraHeaders: Record = {}, +) { + return request.post(`${env.backendUrl}/gateway/llm/chat/completions`, { + headers: { + ...bearer(accessToken), + ...extraHeaders, + }, + data: payload, + }); +} + +export interface LLMGovernanceOverview { + total_managed_instances: number; + non_compliant_count: number; + external_config_count: number; + high_fallback_count: number; + items: Array<{ + instance_id: number; + instance_name: string; + is_compliant: boolean; + }>; +} + +export async function getLLMGovernanceOverview( + request: APIRequestContext, + accessToken: string, +): Promise { + const response = await request.get(`${env.backendUrl}/admin/llm-governance/overview`, { + headers: bearer(accessToken), + }); + return expectOkEnvelope(response); +} + +export interface InstanceSessionUsageResult { + summary: { + total_prompt_tokens: number; + total_completion_tokens: number; + total_tokens: number; + total_estimated_cost: number; + currency: string; + session_count: number; + }; + compliance: { + fallback_session_count: number; + has_fallback_sessions: boolean; + recent_fallback_audit_count: number; + }; + items: Array<{ + session_id: string; + session_key: string; + total_tokens: number; + invocation_count: number; + }>; + total: number; + page: number; + limit: number; +} + +export interface SessionUsageOverviewResult { + summary: { + total_tokens: number; + total_estimated_cost: number; + currency: string; + session_count: number; + }; + items: Array<{ + instance_id: number; + instance_name: string; + instance_type: string; + user_id: number; + summary: { + total_tokens: number; + session_count: number; + total_estimated_cost: number; + currency: string; + }; + compliance: { + fallback_session_count: number; + has_fallback_sessions: boolean; + }; + }>; + total: number; + page: number; + limit: number; +} + +export async function getInstanceSessionUsage( + request: APIRequestContext, + accessToken: string, + instanceId: number, + params?: { page?: number; limit?: number; search?: string; since?: string; until?: string }, +): Promise { + const response = await request.get(`${env.backendUrl}/instances/${instanceId}/session-usage`, { + headers: bearer(accessToken), + params, + }); + return expectOkEnvelope(response); +} + +export async function getAdminSessionUsageOverview( + request: APIRequestContext, + accessToken: string, + params?: { page?: number; limit?: number; search?: string; since?: string; until?: string }, +): Promise { + const response = await request.get(`${env.backendUrl}/admin/session-usage/overview`, { + headers: bearer(accessToken), + params, + }); + return expectOkEnvelope(response); +} diff --git a/e2e/fixtures/dbClient.ts b/e2e/fixtures/dbClient.ts new file mode 100644 index 0000000..cca0c20 --- /dev/null +++ b/e2e/fixtures/dbClient.ts @@ -0,0 +1,23 @@ +import mysql from "mysql2/promise"; +import { env } from "./env.js"; + +export async function getInstanceGatewayToken(instanceId: number): Promise { + const connection = await mysql.createConnection({ + host: env.db.host, + port: env.db.port, + user: env.db.user, + password: env.db.password, + database: env.db.database, + }); + try { + const [rows] = await connection.query<{ access_token: string | null }[]>( + "SELECT access_token FROM instances WHERE id = ? LIMIT 1", + [instanceId], + ); + const row = rows[0]; + const token = row?.access_token?.trim(); + return token ? token : null; + } finally { + await connection.end(); + } +} diff --git a/e2e/tests/instances/llm-governance.spec.ts b/e2e/tests/instances/llm-governance.spec.ts new file mode 100644 index 0000000..c07bf85 --- /dev/null +++ b/e2e/tests/instances/llm-governance.spec.ts @@ -0,0 +1,104 @@ +import { expect, test } from "../../fixtures/test.js"; +import { env } from "../../fixtures/env.js"; +import { login, getLLMGovernanceOverview } from "../../fixtures/apiClient.js"; +import { users } from "../../fixtures/users.js"; +import { execFileSync } from "node:child_process"; + +interface ApiEnvelope { + success: boolean; + data?: T; + error?: string; +} + +function egressProxyOrigin(): string { + return env.backendUrl.replace(/\/api\/v1\/?$/, ""); +} + +test("@p2 create openclaw instance rejects protected env override", async ({ request }) => { + const tokens = await login(request, users.admin); + const suffix = Date.now(); + + const response = await request.post(`${env.backendUrl}/instances`, { + headers: { Authorization: `Bearer ${tokens.access_token}` }, + data: { + name: `e2e-governance-${suffix}`, + type: "openclaw", + mode: "lite", + cpu_cores: 1, + memory_gb: 2, + disk_gb: 20, + gpu_enabled: false, + gpu_count: 0, + os_type: "openclaw", + os_version: "latest", + environment_overrides: { + OPENAI_BASE_URL: "https://api.openai.com/v1", + }, + }, + }); + + expect(response.status()).toBeGreaterThanOrEqual(400); + const body = (await response.json()) as ApiEnvelope; + expect(body.success).toBe(false); + expect(body.error ?? "").toMatch(/managed by the platform/i); +}); + +test("@p2 batch lite create rejects protected env override", async ({ request }) => { + const tokens = await login(request, users.admin); + const suffix = Date.now(); + + const response = await request.post(`${env.backendUrl}/instances/batch/lite`, { + headers: { Authorization: `Bearer ${tokens.access_token}` }, + data: { + name_prefix: `e2e-batch-gov-${suffix}`, + count: 1, + template: { + type: "openclaw", + environment_overrides: { + OPENAI_API_KEY: "sk-test", + }, + }, + }, + }); + + expect(response.status()).toBeGreaterThanOrEqual(400); + const body = (await response.json()) as ApiEnvelope; + expect(body.success).toBe(false); + expect(body.error ?? "").toMatch(/managed by the platform/i); +}); + +test("@p2 admin llm governance overview returns managed runtime summary", async ({ request }) => { + const tokens = await login(request, users.admin); + const overview = await getLLMGovernanceOverview(request, tokens.access_token); + + expect(typeof overview.total_managed_instances).toBe("number"); + expect(Array.isArray(overview.items)).toBe(true); +}); + +test("@p2 @local-only egress proxy blocks direct openai connect", async () => { + let statusCode = ""; + try { + statusCode = execFileSync( + "curl", + [ + "-x", + egressProxyOrigin(), + "-H", + "X-ClawManager-Egress-Instance-Id: 1", + "-m", + "5", + "-s", + "-o", + process.platform === "win32" ? "NUL" : "/dev/null", + "-w", + "%{http_code}", + "https://api.openai.com", + ], + { encoding: "utf8" }, + ).trim(); + } catch { + test.skip(true, "curl unavailable or egress proxy not reachable"); + } + + expect(statusCode).toBe("403"); +}); diff --git a/e2e/tests/instances/session-token-tracking.spec.ts b/e2e/tests/instances/session-token-tracking.spec.ts new file mode 100644 index 0000000..8fbbe22 --- /dev/null +++ b/e2e/tests/instances/session-token-tracking.spec.ts @@ -0,0 +1,167 @@ +import { expect, test } from "../../fixtures/test.js"; +import { env } from "../../fixtures/env.js"; +import { + gatewayChatCompletion, + getInstanceSessionUsage, + listGatewayModels, + listInstances, + login, +} from "../../fixtures/apiClient.js"; +import { getInstanceGatewayToken } from "../../fixtures/dbClient.js"; +import { users } from "../../fixtures/users.js"; + +function firstOpenClawInstance(instances: Awaited>) { + return instances.instances.find( + (instance) => + instance.type === "openclaw" && + instance.status !== "deleting", + ); +} + +async function fetchSessionUsage( + request: Parameters[0], + accessToken: string, + instanceId: number, +) { + return getInstanceSessionUsage(request, accessToken, instanceId, { page: 1, limit: 50 }); +} + +function isGatewaySuccessStatus(status: number): boolean { + return status === 200 || status === 201; +} + +test("@p1 session usage endpoint returns structured payload for openclaw instance", async ({ + request, +}) => { + const accessToken = (await login(request, users.admin)).access_token; + const instances = await listInstances(request, accessToken, { limit: 100 }); + const instance = firstOpenClawInstance(instances); + test.skip(!instance, "No openclaw instance available for session usage test"); + + const data = await fetchSessionUsage(request, accessToken, instance!.id); + expect(data.summary).toBeTruthy(); + expect(Array.isArray(data.items)).toBe(true); + expect(typeof data.total).toBe("number"); + expect(typeof data.compliance.recent_fallback_audit_count).toBe("number"); +}); + +test("@p1 session usage detail requires session_id", async ({ request }) => { + const accessToken = (await login(request, users.admin)).access_token; + const instances = await listInstances(request, accessToken, { limit: 100 }); + const instance = firstOpenClawInstance(instances); + test.skip(!instance, "No openclaw instance available for session usage detail test"); + + const response = await request.get( + `${env.backendUrl}/instances/${instance!.id}/session-usage/detail`, + { + headers: { Authorization: `Bearer ${accessToken}` }, + }, + ); + expect(response.status()).toBe(400); +}); + +test("@p0 gateway calls aggregate tokens by openclaw session key", async ({ request }) => { + const accessToken = (await login(request, users.admin)).access_token; + const instances = await listInstances(request, accessToken, { limit: 100 }); + const instance = firstOpenClawInstance(instances); + test.skip(!instance, "No openclaw instance available for gateway aggregation test"); + + const models = await listGatewayModels(request, accessToken); + test.skip(models.length === 0, "No gateway models configured for session aggregation test"); + + const baseline = await fetchSessionUsage(request, accessToken, instance!.id); + const baselineMain = baseline.items.find((item) => item.session_key === "main"); + let successfulCalls = 0; + + for (let attempt = 0; attempt < 3; attempt += 1) { + const response = await gatewayChatCompletion(request, accessToken, { + model: "auto", + instance_id: instance!.id, + messages: [{ role: "user", content: `session aggregation probe ${Date.now()}-${attempt}` }], + }, { + "x-openclaw-session-key": "main", + "x-openclaw-run-id": `e2e-session-${Date.now()}-${attempt}`, + }); + if (isGatewaySuccessStatus(response.status())) { + successfulCalls += 1; + } + } + test.skip(successfulCalls === 0, "Gateway upstream unavailable; skipping token aggregation assertion"); + + await expect + .poll(async () => { + const latest = await fetchSessionUsage(request, accessToken, instance!.id); + const main = latest.items.find((item) => item.session_key === "main"); + return main?.invocation_count ?? 0; + }, { timeout: 20_000 }) + .toBeGreaterThan(baselineMain?.invocation_count ?? 0); + + await expect + .poll(async () => { + const latest = await fetchSessionUsage(request, accessToken, instance!.id); + const main = latest.items.find((item) => item.session_key === "main"); + return main?.total_tokens ?? 0; + }, { timeout: 20_000 }) + .toBeGreaterThan(baselineMain?.total_tokens ?? 0); +}); + +test("@p0 gateway missing session key surfaces fallback compliance", async ({ request }) => { + const accessToken = (await login(request, users.admin)).access_token; + const instances = await listInstances(request, accessToken, { limit: 100 }); + const instance = firstOpenClawInstance(instances); + test.skip(!instance, "No openclaw instance available for fallback compliance test"); + + const models = await listGatewayModels(request, accessToken); + test.skip(models.length === 0, "No gateway models configured for fallback compliance test"); + + const baseline = await fetchSessionUsage(request, accessToken, instance!.id); + const response = await gatewayChatCompletion(request, accessToken, { + model: "auto", + instance_id: instance!.id, + messages: [{ role: "user", content: `fallback probe ${Date.now()}` }], + }, { + "x-openclaw-run-id": `e2e-fallback-${Date.now()}`, + }); + test.skip(!isGatewaySuccessStatus(response.status()), "Gateway upstream unavailable; skipping fallback assertion"); + + await expect + .poll(async () => { + const latest = await fetchSessionUsage(request, accessToken, instance!.id); + return latest.compliance.has_fallback_sessions; + }, { timeout: 20_000 }) + .toBe(true); + + await expect + .poll(async () => { + const latest = await fetchSessionUsage(request, accessToken, instance!.id); + return latest.compliance.recent_fallback_audit_count; + }, { timeout: 20_000 }) + .toBeGreaterThan(baseline.compliance.recent_fallback_audit_count); +}); + +test("@p0 instance gateway token can call chat completions", async ({ request }) => { + const accessToken = (await login(request, users.admin)).access_token; + const instances = await listInstances(request, accessToken, { limit: 100 }); + const instance = firstOpenClawInstance(instances); + test.skip(!instance, "No openclaw instance available for instance gateway token test"); + + let gatewayToken: string | null = null; + try { + gatewayToken = await getInstanceGatewayToken(instance!.id); + } catch { + test.skip(true, "E2E database unavailable for instance gateway token lookup"); + } + test.skip(!gatewayToken, "Instance gateway token not provisioned"); + + const models = await listGatewayModels(request, gatewayToken); + test.skip(models.length === 0, "No gateway models configured for instance token test"); + + const response = await gatewayChatCompletion(request, gatewayToken, { + model: "auto", + messages: [{ role: "user", content: `instance token probe ${Date.now()}` }], + }, { + "x-openclaw-session-key": "main", + "x-openclaw-run-id": `e2e-instance-token-${Date.now()}`, + }); + expect([200, 201]).toContain(response.status()); +}); diff --git a/e2e/tests/instances/session-usage-admin.spec.ts b/e2e/tests/instances/session-usage-admin.spec.ts new file mode 100644 index 0000000..4d63c4c --- /dev/null +++ b/e2e/tests/instances/session-usage-admin.spec.ts @@ -0,0 +1,95 @@ +import { expect, test } from "../../fixtures/test.js"; +import { env } from "../../fixtures/env.js"; +import { + getAdminSessionUsageOverview, + getInstanceSessionUsage, + listInstances, + login, + registerUser, +} from "../../fixtures/apiClient.js"; +import { users } from "../../fixtures/users.js"; + +function firstManagedRuntimeInstance(instances: Awaited>) { + return instances.instances.find( + (instance) => + (instance.type === "openclaw" || instance.type === "hermes") && + instance.status !== "deleting", + ); +} + +test("@p1 admin session usage overview returns structured payload", async ({ request }) => { + const accessToken = (await login(request, users.admin)).access_token; + const overview = await getAdminSessionUsageOverview(request, accessToken, { + page: 1, + limit: 20, + }); + + expect(overview.summary).toBeTruthy(); + expect(Array.isArray(overview.items)).toBe(true); + expect(typeof overview.total).toBe("number"); + expect(typeof overview.page).toBe("number"); + expect(typeof overview.limit).toBe("number"); +}); + +test("@p1 admin session usage overview rejects invalid since timestamp", async ({ request }) => { + const accessToken = (await login(request, users.admin)).access_token; + const response = await request.get(`${env.backendUrl}/admin/session-usage/overview`, { + headers: { Authorization: `Bearer ${accessToken}` }, + params: { since: "not-a-date" }, + }); + expect(response.status()).toBe(400); +}); + +test("@p1 instance session usage accepts since query parameter", async ({ request }) => { + const accessToken = (await login(request, users.admin)).access_token; + const instances = await listInstances(request, accessToken, { limit: 100 }); + const instance = firstManagedRuntimeInstance(instances); + test.skip(!instance, "No openclaw/hermes instance available for since-filter test"); + + const since = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + const data = await getInstanceSessionUsage(request, accessToken, instance!.id, { + page: 1, + limit: 20, + since, + }); + + expect(data.summary).toBeTruthy(); + expect(Array.isArray(data.items)).toBe(true); +}); + +test("@p1 instance session usage rejects invalid since timestamp", async ({ request }) => { + const accessToken = (await login(request, users.admin)).access_token; + const instances = await listInstances(request, accessToken, { limit: 100 }); + const instance = firstManagedRuntimeInstance(instances); + test.skip(!instance, "No openclaw/hermes instance available for invalid since test"); + + const response = await request.get( + `${env.backendUrl}/instances/${instance!.id}/session-usage`, + { + headers: { Authorization: `Bearer ${accessToken}` }, + params: { since: "bad-timestamp" }, + }, + ); + expect(response.status()).toBe(400); +}); + +test("@p2 non-admin cannot access session usage overview", async ({ request }) => { + await registerUser(request, users.user); + const accessToken = (await login(request, users.user)).access_token; + const response = await request.get(`${env.backendUrl}/admin/session-usage/overview`, { + headers: { Authorization: `Bearer ${accessToken}` }, + }); + expect(response.status()).toBe(403); +}); + +test("@p1 session usage rejects until before since", async ({ request }) => { + const accessToken = (await login(request, users.admin)).access_token; + const response = await request.get(`${env.backendUrl}/admin/session-usage/overview`, { + headers: { Authorization: `Bearer ${accessToken}` }, + params: { + since: "2026-07-10T00:00:00Z", + until: "2026-07-01T00:00:00Z", + }, + }); + expect(response.status()).toBe(400); +}); diff --git a/frontend/src/components/AdminLayout.tsx b/frontend/src/components/AdminLayout.tsx index 62a0be1..77f5eb8 100644 --- a/frontend/src/components/AdminLayout.tsx +++ b/frontend/src/components/AdminLayout.tsx @@ -74,7 +74,7 @@ const AdminLayout: React.FC = ({ children, title = '' }) => { path: '/admin/ai-gateway', label: t('nav.aiGateway'), icon: Bot, - matchPaths: ['/admin/models', '/admin/ai-audit', '/admin/costs', '/admin/risk-rules'], + matchPaths: ['/admin/models', '/admin/ai-audit', '/admin/costs', '/admin/risk-rules', '/admin/session-usage'], }, { path: '/admin/settings', label: t('nav.settings'), icon: Settings }, ]; diff --git a/frontend/src/components/InstanceCollapsiblePanel.tsx b/frontend/src/components/InstanceCollapsiblePanel.tsx new file mode 100644 index 0000000..dba8726 --- /dev/null +++ b/frontend/src/components/InstanceCollapsiblePanel.tsx @@ -0,0 +1,106 @@ +import React, { useEffect, useState } from "react"; +import { ChevronDown, ChevronUp } from "lucide-react"; +import { useI18n } from "../contexts/I18nContext"; + +type Props = { + storageKey: string; + title: string; + icon: React.ReactNode; + defaultCollapsed?: boolean; + summary?: React.ReactNode; + headerActions?: React.ReactNode; + onExpandedChange?: (expanded: boolean) => void; + contentClassName?: string; + children: React.ReactNode; +}; + +function readStoredCollapsed(storageKey: string, defaultCollapsed: boolean): boolean { + try { + const stored = localStorage.getItem(storageKey); + if (stored === "true") { + return true; + } + if (stored === "false") { + return false; + } + } catch { + // ignore storage failures + } + return defaultCollapsed; +} + +export default function InstanceCollapsiblePanel({ + storageKey, + title, + icon, + defaultCollapsed = true, + summary, + headerActions, + onExpandedChange, + contentClassName, + children, +}: Props) { + const { t } = useI18n(); + const [collapsed, setCollapsed] = useState(() => readStoredCollapsed(storageKey, defaultCollapsed)); + + const toggle = () => { + setCollapsed((current) => { + const next = !current; + onExpandedChange?.(!next); + return next; + }); + }; + + useEffect(() => { + onExpandedChange?.(!collapsed); + }, [collapsed, onExpandedChange]); + + useEffect(() => { + try { + localStorage.setItem(storageKey, String(collapsed)); + } catch { + // ignore storage failures + } + }, [collapsed, storageKey]); + + return ( +
+
+ +
+ {!collapsed ? headerActions : null} + +
+
+ {!collapsed ? ( +
{children}
+ ) : null} +
+ ); +} diff --git a/frontend/src/components/InstanceServiceFrame.tsx b/frontend/src/components/InstanceServiceFrame.tsx index 94697c9..e48c258 100644 --- a/frontend/src/components/InstanceServiceFrame.tsx +++ b/frontend/src/components/InstanceServiceFrame.tsx @@ -105,7 +105,7 @@ export function InstanceServiceFrame({ const renderFrameShell = (content: ReactNode) => (
diff --git a/frontend/src/components/InstanceSessionUsagePanel.tsx b/frontend/src/components/InstanceSessionUsagePanel.tsx new file mode 100644 index 0000000..eab29d8 --- /dev/null +++ b/frontend/src/components/InstanceSessionUsagePanel.tsx @@ -0,0 +1,468 @@ +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { BarChart3, ChevronDown, ChevronUp, Download, RefreshCw, Search } from "lucide-react"; +import { useI18n } from "../contexts/I18nContext"; +import InstanceCollapsiblePanel from "./InstanceCollapsiblePanel"; +import { instanceService } from "../services/instanceService"; +import type { + InstanceSessionUsageDetail, + InstanceSessionUsageItem, + InstanceSessionUsageResult, +} from "../types/instance"; +import { + buildSessionUsageCsv, + downloadSessionUsageCsv, + resolveSessionUsageSince, + type SessionUsageTimeRange, +} from "../utils/sessionUsageExport"; + +type Props = { + instanceId: number; + instanceType: string; + onPanelExpandedChange?: (expanded: boolean) => void; +}; + +const PAGE_SIZE = 10; +const AUTO_REFRESH_MS = 15000; + +function formatNumber(value: number) { + return new Intl.NumberFormat().format(value); +} + +function formatCost(value: number, currency: string) { + return new Intl.NumberFormat(undefined, { + style: "currency", + currency: currency || "USD", + maximumFractionDigits: 4, + }).format(value); +} + +function formatDateTime(value: string | undefined, locale: string) { + if (!value) { + return "-"; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return value; + } + return date.toLocaleString(locale); +} + +export default function InstanceSessionUsagePanel({ + instanceId, + instanceType, + onPanelExpandedChange, +}: Props) { + const { t, locale } = useI18n(); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(""); + const [search, setSearch] = useState(""); + const [debouncedSearch, setDebouncedSearch] = useState(""); + const [page, setPage] = useState(1); + const [timeRange, setTimeRange] = useState("all"); + const [autoRefresh, setAutoRefresh] = useState(false); + const [exporting, setExporting] = useState(false); + const [result, setResult] = useState(null); + const [expandedSessionId, setExpandedSessionId] = useState(null); + const [detail, setDetail] = useState(null); + const [detailLoading, setDetailLoading] = useState(false); + const [detailError, setDetailError] = useState(""); + const [panelExpanded, setPanelExpanded] = useState(false); + + const handlePanelExpandedChange = useCallback( + (expanded: boolean) => { + setPanelExpanded(expanded); + onPanelExpandedChange?.(expanded); + }, + [onPanelExpandedChange], + ); + + const supported = instanceType === "openclaw" || instanceType === "hermes"; + const since = useMemo(() => resolveSessionUsageSince(timeRange), [timeRange]); + + useEffect(() => { + const timer = window.setTimeout(() => setDebouncedSearch(search.trim()), 300); + return () => window.clearTimeout(timer); + }, [search]); + + const loadDetail = useCallback( + async (sessionId: string, options?: { silent?: boolean }) => { + if (!options?.silent) { + setDetailLoading(true); + } + setDetailError(""); + try { + const data = await instanceService.getInstanceSessionUsageDetail(instanceId, sessionId, { + since, + }); + setDetail(data); + } catch (err) { + setDetail(null); + setDetailError(err instanceof Error ? err.message : t("instances.sessionUsage.detailLoadFailed")); + } finally { + if (!options?.silent) { + setDetailLoading(false); + } + } + }, + [instanceId, since, t], + ); + + const loadUsage = useCallback(async (options?: { silent?: boolean }) => { + if (!supported) { + return; + } + if (!options?.silent) { + setLoading(true); + } + setError(""); + try { + const data = await instanceService.getInstanceSessionUsage(instanceId, { + page, + limit: PAGE_SIZE, + search: debouncedSearch || undefined, + since, + }); + setResult(data); + } catch (err) { + setError(err instanceof Error ? err.message : t("instances.sessionUsage.loadFailed")); + } finally { + if (!options?.silent) { + setLoading(false); + } + } + }, [debouncedSearch, instanceId, page, since, supported, t]); + + useEffect(() => { + void loadUsage(); + }, [loadUsage]); + + useEffect(() => { + setPage(1); + }, [debouncedSearch, timeRange]); + + useEffect(() => { + setExpandedSessionId(null); + setDetail(null); + setDetailError(""); + }, [debouncedSearch, timeRange]); + + useEffect(() => { + if (!autoRefresh || !supported || !panelExpanded) { + return; + } + const timer = window.setInterval(() => { + void (async () => { + await loadUsage({ silent: true }); + if (expandedSessionId) { + await loadDetail(expandedSessionId, { silent: true }); + } + })(); + }, AUTO_REFRESH_MS); + return () => window.clearInterval(timer); + }, [autoRefresh, expandedSessionId, loadDetail, loadUsage, panelExpanded, supported]); + + const totalPages = useMemo(() => { + if (!result) { + return 1; + } + return Math.max(1, Math.ceil(result.total / PAGE_SIZE)); + }, [result]); + + const exportCsv = async () => { + setExporting(true); + try { + const items: InstanceSessionUsageItem[] = []; + let pageCursor = 1; + let total = 0; + let currency = "USD"; + do { + const batch = await instanceService.getInstanceSessionUsage(instanceId, { + page: pageCursor, + limit: 100, + search: debouncedSearch || undefined, + since, + }); + items.push(...batch.items); + total = batch.total; + currency = batch.summary.currency || currency; + pageCursor += 1; + } while (items.length < total); + + const csv = buildSessionUsageCsv(items, currency); + downloadSessionUsageCsv(csv, `instance-${instanceId}-session-usage.csv`); + } catch (err) { + setError(err instanceof Error ? err.message : t("instances.sessionUsage.exportFailed")); + } finally { + setExporting(false); + } + }; + + const toggleDetail = async (item: InstanceSessionUsageItem) => { + if (expandedSessionId === item.session_id) { + setExpandedSessionId(null); + setDetail(null); + setDetailError(""); + return; + } + setExpandedSessionId(item.session_id); + setDetail(null); + await loadDetail(item.session_id); + }; + + const sessionPanelSummary = result + ? t("instances.sessionUsage.panelSummary", { + sessions: formatNumber(result.summary.session_count), + tokens: formatNumber(result.summary.total_tokens), + cost: formatCost(result.summary.total_estimated_cost, result.summary.currency), + }) + : loading + ? t("instances.sessionUsage.loading") + : t("instances.sessionUsage.empty"); + + if (!supported) { + return null; + } + + return ( + } + summary={sessionPanelSummary} + onExpandedChange={handlePanelExpandedChange} + headerActions={ +
+
+ + setSearch(event.target.value)} + placeholder={t("instances.sessionUsage.sessionKey")} + className="w-full rounded-md border border-slate-200 py-2 pl-8 pr-3 text-sm" + /> +
+ + + + +
+ } + > +

{t("instances.sessionUsage.gatewayOnlyNotice")}

+ + {result?.compliance.has_fallback_sessions && ( +
+ {t("instances.sessionUsage.fallbackWarning")} +
+ )} + + {result && ( +
+
+
{t("instances.sessionUsage.total")}
+
+ {formatNumber(result.summary.total_tokens)} +
+
+
+
{t("instances.sessionUsage.cost")}
+
+ {formatCost(result.summary.total_estimated_cost, result.summary.currency)} +
+
+
+
{t("instances.sessionUsage.sessions")}
+
+ {formatNumber(result.summary.session_count)} +
+
+
+
{t("instances.sessionUsage.summary")}
+
+ {formatNumber(result.summary.total_prompt_tokens)} /{" "} + {formatNumber(result.summary.total_completion_tokens)} +
+
+
+ )} + + {error && ( +
+ {error} + +
+ )} + + {loading ? ( +
+ {t("instances.sessionUsage.loading")} +
+ ) : !result || result.items.length === 0 ? ( +
+ {t("instances.sessionUsage.empty")} +
+ ) : ( +
+ + + + + + + + + + + + + + {result.items.map((item) => { + const expanded = expandedSessionId === item.session_id; + return ( + + + + + + + + + + + + {expanded && ( + + + + )} + + ); + })} + +
{t("instances.sessionUsage.sessionKey")}{t("instances.sessionUsage.prompt")}{t("instances.sessionUsage.completion")}{t("instances.sessionUsage.total")}{t("instances.sessionUsage.cost")}{t("instances.sessionUsage.invocations")}{t("instances.sessionUsage.lastActive")} +
+
{item.session_key}
+ {item.title &&
{item.title}
} + 		{formatNumber(item.prompt_tokens)}{formatNumber(item.completion_tokens)}{formatNumber(item.total_tokens)}{formatCost(item.estimated_cost, item.currency)}{formatNumber(item.invocation_count)}{formatDateTime(item.last_seen_at, locale)} + +
+ {detailLoading ? ( +
{t("instances.sessionUsage.detailLoading")}
+ ) : detail ? ( +
+
+
+ {t("instances.sessionUsage.modelBreakdown")} +
+
+ {detail.model_breakdown.map((row) => ( +
+ {row.label} + + {formatNumber(row.total_tokens)} tokens ·{" "} + {formatCost(row.estimated_cost, detail.currency)} + +
+ ))} +
+
+
+
+ {t("instances.sessionUsage.recentTraces")} +
+
+ {detail.recent_traces.map((trace) => ( +
+
{trace.trace_id}
+
+ {trace.requested_model} · {trace.status} ·{" "} + {formatNumber(trace.total_tokens)} tokens +
+
+ ))} +
+
+
+ ) : ( +
+ {detailError || t("instances.sessionUsage.detailEmpty")} +
+ )} +
+
+ )} + + {result && result.total > PAGE_SIZE && ( +
+ + {t("instances.sessionUsage.pageSummary", { page, totalPages })} + +
+ + +
+
+ )} + + ); +} diff --git a/frontend/src/components/InstanceSkillHubPanel.tsx b/frontend/src/components/InstanceSkillHubPanel.tsx new file mode 100644 index 0000000..78a2f45 --- /dev/null +++ b/frontend/src/components/InstanceSkillHubPanel.tsx @@ -0,0 +1,960 @@ +import React, { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { Link } from "react-router-dom"; +import { KeyRound, Plus, X } from "lucide-react"; +import { useAuth } from "../contexts/AuthContext"; +import { useI18n } from "../contexts/I18nContext"; +import InstanceCollapsiblePanel from "./InstanceCollapsiblePanel"; +import { instanceService } from "../services/instanceService"; +import { skillHubService } from "../services/skillHubService"; +import { skillService } from "../services/skillService"; +import type { Instance, InstanceRuntimeDetails } from "../types/instance"; +import type { InstanceSkill, Skill, SkillHubTag } from "../types/skill"; + +const SKILLS_POLL_INTERVAL_MS = 8000; +const SKILLS_BURST_POLL_INTERVAL_MS = 3000; +const SKILLS_BURST_WINDOW_MS = 60000; +const SKILL_SYNC_POLL_MS = 2000; +const SKILL_SYNC_TIMEOUT_MS = 60000; +const INSTANCE_SKILL_PAGE_SIZE = 5; + +type TranslateFn = (key: string, variables?: Record) => string; + +function isHubInstalledSkill(item: InstanceSkill): boolean { + return (item.source_type || "").toLowerCase() === "injected_by_clawmanager"; +} + +function resolveInstanceSkillProvenance(item: InstanceSkill): "native" | "hub_installed" { + return isHubInstalledSkill(item) ? "hub_installed" : "native"; +} + +function isNativeInstanceSkill(item: InstanceSkill): boolean { + return !isHubInstalledSkill(item); +} + +function fingerprintNativeSkills(skills: InstanceSkill[]): string { + return skills + .filter(isNativeInstanceSkill) + .map( + (item) => + `${item.skill_id}:${item.last_seen_at || ""}:${item.skill?.source_type || ""}:${item.skill?.updated_at || ""}`, + ) + .sort() + .join("|"); +} + +function countNativeSkills(skills: InstanceSkill[]): number { + return skills.filter(isNativeInstanceSkill).length; +} + +function isSkillSyncCommandFinished(status: string): boolean { + return status === "succeeded" || status === "failed" || status === "timed_out"; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => { + window.setTimeout(resolve, ms); + }); +} + +function paginateInstanceSkills(skills: InstanceSkill[], page: number) { + const totalPages = Math.max(1, Math.ceil(skills.length / INSTANCE_SKILL_PAGE_SIZE)); + const currentPage = Math.min(page, totalPages); + return { + items: skills.slice( + (currentPage - 1) * INSTANCE_SKILL_PAGE_SIZE, + currentPage * INSTANCE_SKILL_PAGE_SIZE, + ), + totalPages, + currentPage, + }; +} + +function matchesSkillSearch(query: string, skill?: Skill, fallbackId?: string | number): boolean { + const needle = query.trim().toLowerCase(); + if (!needle) { + return true; + } + const haystacks = [ + skill?.name, + skill?.skill_key, + skill?.owner_username, + skill?.description, + fallbackId != null ? String(fallbackId) : "", + ]; + return haystacks.some((value) => (value || "").toLowerCase().includes(needle)); +} + +function matchesInstanceSkillSearch(item: InstanceSkill, query: string): boolean { + return matchesSkillSearch(query, item.skill, item.skill_id); +} + +function formatDateTime(value: string | undefined, locale: string) { + if (!value) { + return "-"; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return value; + } + return date.toLocaleString(locale); +} + +function skillRiskLabel(t: TranslateFn, riskLevel?: string | null) { + switch ((riskLevel || "").toLowerCase()) { + case "none": + return t("instances.skillRiskNone"); + case "low": + return t("instances.skillRiskLow"); + case "medium": + return t("instances.skillRiskMedium"); + case "high": + return t("instances.skillRiskHigh"); + default: + return t("instances.skillRiskUnknown"); + } +} + +type InstanceSkillCardProps = { + item: InstanceSkill; + t: TranslateFn; + locale: string; + userId?: number; + actionLoading: string | null; + allowImportToLibrary: boolean; + onImportToLibrary: (skillId: number) => void; + onRetryPackageCollect: (skillId: number) => void; + onPublish: (skillId: number) => void; + onRemove: (skillId: number) => void; + shouldShowImportToLibrary: (skill: Skill) => boolean; + isSkillPackagePending: (skill?: Skill) => boolean; + isSkillPackageCollectFailed: (skill?: Skill) => boolean; + blockReasonLabel: (reason?: string) => string | null; + isLiteInstance?: boolean; +}; + +function InstanceSkillCard({ + item, + t, + locale, + userId, + actionLoading, + allowImportToLibrary, + onImportToLibrary, + onRetryPackageCollect, + onPublish, + onRemove, + shouldShowImportToLibrary, + isSkillPackagePending, + isSkillPackageCollectFailed, + blockReasonLabel, + isLiteInstance = false, +}: InstanceSkillCardProps) { + const provenance = + resolveInstanceSkillProvenance(item) === "hub_installed" + ? t("instances.skillProvenanceInjected") + : t("instances.skillProvenanceNative"); + + return ( +
+
+
+
+ + {item.skill?.name || t("instances.skillFallback", { id: item.skill_id })} + + + {provenance} + + + {skillRiskLabel(t, item.skill?.risk_level)} + + {isSkillPackagePending(item.skill) ? ( + + {t("instances.skillPackageSyncing")} + + ) : null} + {isSkillPackageCollectFailed(item.skill) ? ( + + {t("instances.skillPackageCollectFailed")} + + ) : null} +
+

+ {item.skill?.skill_key || item.skill_id} + {item.last_seen_at + ? ` · ${t("instances.lastSeenAt", { value: formatDateTime(item.last_seen_at, locale) })}` + : ""} +

+ {item.skill?.package_collect_error ? ( +

{item.skill.package_collect_error}

+ ) : null} + {item.skill?.publish_blocked_reason && !item.skill.publishable ? ( +

{blockReasonLabel(item.skill.publish_blocked_reason)}

+ ) : null} +
+
+ {isSkillPackageCollectFailed(item.skill) ? ( + + ) : null} + {allowImportToLibrary && + isNativeInstanceSkill(item) && + item.skill && + item.skill.user_id === userId && + item.skill.source_type === "discovered" && + shouldShowImportToLibrary(item.skill) ? ( + + ) : null} + {item.skill && + isNativeInstanceSkill(item) && + item.skill.user_id === userId && + item.skill.source_type === "uploaded" && + item.skill.visibility !== "public" && + item.skill.publishable ? ( + + ) : null} + +
+
+
+ ); +} + +interface InstanceSkillHubPanelProps { + instance: Instance; + onRuntimeDetailsChange: (details: InstanceRuntimeDetails) => void; + onPanelExpandedChange?: (expanded: boolean) => void; +} + +const InstanceSkillHubPanel: React.FC = ({ + instance, + onRuntimeDetailsChange, + onPanelExpandedChange, +}) => { + const { t, locale } = useI18n(); + const { user } = useAuth(); + const instanceId = instance.id; + + const [skillLoading, setSkillLoading] = useState(false); + const [skillError, setSkillError] = useState(null); + const [actionLoading, setActionLoading] = useState(null); + const [skillsBurstUntil, setSkillsBurstUntil] = useState(0); + const [skillSyncPhase, setSkillSyncPhase] = useState<"idle" | "syncing" | "success" | "failed">("idle"); + const [skillSyncDetail, setSkillSyncDetail] = useState(""); + const [lastSkillSyncAt, setLastSkillSyncAt] = useState(null); + const skillSyncActiveRef = useRef(false); + const [instanceSkills, setInstanceSkills] = useState([]); + const [availableSkills, setAvailableSkills] = useState([]); + const [hubTags, setHubTags] = useState([]); + const [publishSkillId, setPublishSkillId] = useState(null); + const [selectedHubTagIds, setSelectedHubTagIds] = useState([]); + const [instanceSkillPage, setInstanceSkillPage] = useState(1); + const [nativeSkillSearch, setNativeSkillSearch] = useState(""); + const [hubCatalogSearch, setHubCatalogSearch] = useState(""); + const [panelExpanded, setPanelExpanded] = useState(false); + + const handlePanelExpandedChange = useCallback( + (expanded: boolean) => { + setPanelExpanded(expanded); + onPanelExpandedChange?.(expanded); + }, + [onPanelExpandedChange], + ); + + const skillsPollInterval = + skillsBurstUntil > Date.now() ? SKILLS_BURST_POLL_INTERVAL_MS : SKILLS_POLL_INTERVAL_MS; + + const reloadSkillSection = useCallback(async () => { + const [instanceSkillItems, catalog, tagItems] = await Promise.all([ + skillService.listInstanceSkills(instanceId), + skillHubService.listCatalog({ page: 1, page_size: 1000 }), + skillHubService.listTags(), + ]); + setInstanceSkills(instanceSkillItems); + setHubTags(tagItems); + setAvailableSkills( + (catalog.items || []).filter( + (item) => + item.status === "active" && + (item.visibility || "").toLowerCase() === "public" && + item.risk_level !== "medium" && + item.risk_level !== "high", + ), + ); + }, [instanceId]); + + const refreshSkills = useCallback(async () => { + const items = await skillService.listInstanceSkills(instanceId); + setInstanceSkills(items); + }, [instanceId]); + + useEffect(() => { + let disposed = false; + const load = async () => { + try { + setSkillLoading(true); + await reloadSkillSection(); + if (!disposed) { + setSkillError(null); + } + } catch (err: unknown) { + if (!disposed) { + const message = + (err as { response?: { data?: { error?: string } } })?.response?.data?.error || + "Failed to load skills"; + setSkillError(message); + } + } finally { + if (!disposed) { + setSkillLoading(false); + } + } + }; + void load(); + return () => { + disposed = true; + }; + }, [reloadSkillSection]); + + useEffect(() => { + skillSyncActiveRef.current = false; + setSkillSyncPhase("idle"); + setSkillSyncDetail(""); + setNativeSkillSearch(""); + setHubCatalogSearch(""); + }, [instanceId]); + + useEffect(() => { + if (!panelExpanded) { + return; + } + const skillsTimer = window.setInterval(() => { + if (!document.hidden) { + void refreshSkills(); + } + }, skillsPollInterval); + return () => window.clearInterval(skillsTimer); + }, [panelExpanded, refreshSkills, skillsPollInterval]); + + useEffect(() => { + setInstanceSkillPage(1); + }, [instanceId]); + + useEffect(() => { + setInstanceSkillPage(1); + }, [nativeSkillSearch]); + + const filteredInstanceSkills = useMemo(() => { + const query = nativeSkillSearch.trim(); + if (!query) { + return instanceSkills; + } + return instanceSkills.filter((item) => matchesInstanceSkillSearch(item, query)); + }, [instanceSkills, nativeSkillSearch]); + + const instanceSkillsPagination = useMemo( + () => paginateInstanceSkills(filteredInstanceSkills, instanceSkillPage), + [filteredInstanceSkills, instanceSkillPage], + ); + const hubCatalogRows = useMemo(() => { + const installedBySkillId = new Map(instanceSkills.map((item) => [item.skill_id, item])); + return availableSkills.map((skill) => ({ + skill, + installed: installedBySkillId.has(skill.id), + instanceSkill: installedBySkillId.get(skill.id), + })); + }, [availableSkills, instanceSkills]); + const filteredHubCatalogRows = useMemo(() => { + const query = hubCatalogSearch.trim(); + if (!query) { + return hubCatalogRows; + } + return hubCatalogRows.filter(({ skill }) => matchesSkillSearch(query, skill, skill.id)); + }, [hubCatalogRows, hubCatalogSearch]); + + const hubErrorMessage = (err: unknown, fallback: string) => { + const errorKey = (err as { response?: { data?: { error?: string } } })?.response?.data?.error; + if (!errorKey) { + return fallback; + } + const labelKey = `skillHubPage.blockReasons.${errorKey}`; + const label = t(labelKey); + return label === labelKey ? errorKey : label; + }; + + const shouldShowImportToLibrary = (skill: Skill) => + skill.status !== "deleted" && + (skill.source_type !== "uploaded" || + skill.publish_blocked_reason === "skill_package_pending" || + skill.publish_blocked_reason === "skill_package_materializing"); + + const isSkillPackagePending = (skill?: Skill) => + skill?.publish_blocked_reason === "skill_package_pending" || + skill?.publish_blocked_reason === "skill_package_materializing" || + skill?.package_materialize_status === "pending" || + skill?.package_materialize_status === "running"; + + const isSkillPackageCollectFailed = (skill?: Skill) => + skill?.publish_blocked_reason === "skill_package_collect_failed" || + skill?.publish_blocked_reason === "skill_package_materialize_failed" || + skill?.package_materialize_status === "failed"; + + const blockReasonLabel = (reason?: string) => { + if (!reason) { + return null; + } + const labelKey = `skillHubPage.blockReasons.${reason}`; + const label = t(labelKey); + return label === labelKey ? reason : label; + }; + + const hubTagLabel = (tag: SkillHubTag) => t(`skillHubPage.tags.${tag.tag_key}`) || tag.name; + + const isLiteInstance = + instance.instance_mode === "lite" || instance.runtime_type === "gateway"; + + const usesWorkspaceSkillSync = + isLiteInstance || + (Boolean(instance.workspace_path?.trim()) && + (instance.type === "hermes" || instance.type === "openclaw")); + + const handleInstallHubSkill = async (skillId: number) => { + if (!usesWorkspaceSkillSync) { + const runtimeSnapshot = await instanceService.getRuntimeDetails(instanceId); + const agentStatus = + runtimeSnapshot.agent?.status || runtimeSnapshot.runtime?.agent_status || "offline"; + if (agentStatus === "offline") { + setSkillError(t("instances.installSkillRequiresAgent")); + return; + } + } + try { + setActionLoading(`install-hub-${skillId}`); + await skillService.attachSkillToInstance(instanceId, skillId); + setSkillsBurstUntil(Date.now() + SKILLS_BURST_WINDOW_MS); + await reloadSkillSection(); + } catch (err: unknown) { + setSkillError(hubErrorMessage(err, t("instances.failedToAttachSkill"))); + } finally { + setActionLoading(null); + } + }; + + const handleRemoveSkill = async (skillId: number) => { + try { + setActionLoading(`remove-skill-${skillId}`); + await skillService.removeSkillFromInstance(instanceId, skillId); + await reloadSkillSection(); + } catch (err: unknown) { + setSkillError(hubErrorMessage(err, t("instances.failedToRemoveSkill"))); + } finally { + setActionLoading(null); + } + }; + + const handleRetryPackageCollect = async (skillId: number) => { + try { + setActionLoading(`retry-package-${skillId}`); + await skillHubService.retryPackageCollect(instanceId, skillId); + setSkillsBurstUntil(Date.now() + SKILLS_BURST_WINDOW_MS); + await reloadSkillSection(); + } catch (err: unknown) { + alert(hubErrorMessage(err, t("instances.skillPackageCollectFailed"))); + } finally { + setActionLoading(null); + } + }; + + const handleImportSkillToLibrary = async (skillId: number) => { + try { + setActionLoading(`import-library-${skillId}`); + await skillHubService.importInstanceSkill(instanceId, skillId); + await reloadSkillSection(); + } catch (err: unknown) { + const errorKey = (err as { response?: { data?: { error?: string } } })?.response?.data?.error; + if (errorKey === "skill_package_pending") { + await reloadSkillSection(); + return; + } + alert(hubErrorMessage(err, t("skillHubPage.importToLibraryFailed"))); + } finally { + setActionLoading(null); + } + }; + + const handlePublishSkillToHub = async () => { + if (publishSkillId === null || selectedHubTagIds.length === 0) { + alert(t("skillHubPage.errors.tagsRequired")); + return; + } + try { + setActionLoading(`publish-hub-${publishSkillId}`); + await skillHubService.publishFromInstance(instanceId, publishSkillId, selectedHubTagIds); + setPublishSkillId(null); + setSelectedHubTagIds([]); + await reloadSkillSection(); + } catch (err: unknown) { + alert(hubErrorMessage(err, t("skillHubPage.errors.publish"))); + } finally { + setActionLoading(null); + } + }; + + const handleSyncInstanceSkills = async () => { + if (skillSyncPhase === "syncing") { + return; + } + + skillSyncActiveRef.current = true; + setSkillSyncPhase("syncing"); + setSkillSyncDetail(t("instances.syncSkillsStatusRequesting")); + + const baselineFingerprint = fingerprintNativeSkills(instanceSkills); + const baselineCount = countNativeSkills(instanceSkills); + + try { + if (!usesWorkspaceSkillSync) { + const runtimeSnapshot = await instanceService.getRuntimeDetails(instanceId); + const agentStatus = + runtimeSnapshot.agent?.status || runtimeSnapshot.runtime?.agent_status || "offline"; + if (agentStatus === "offline") { + setSkillSyncPhase("failed"); + setSkillSyncDetail(t("instances.syncSkillsAgentOffline")); + return; + } + } + + const command = await instanceService.syncInstanceSkills(instanceId); + setSkillsBurstUntil(Date.now() + SKILLS_BURST_WINDOW_MS); + setSkillSyncDetail(t("instances.syncSkillsStatusAgent", { status: command.status })); + + const deadline = Date.now() + SKILL_SYNC_TIMEOUT_MS; + let latestSkills = instanceSkills; + let resolved = false; + let syncFailed = false; + let resolvedSuccessDetail: string | null = null; + let fingerprintChanged = false; + + while (Date.now() < deadline && skillSyncActiveRef.current) { + const [runtimeData, skillItems] = await Promise.all([ + instanceService.getRuntimeDetails(instanceId), + skillService.listInstanceSkills(instanceId), + ]); + onRuntimeDetailsChange(runtimeData); + setInstanceSkills(skillItems); + latestSkills = skillItems; + + const trackedCommand = + runtimeData.commands.find((item) => item.id === command.id) || command; + const nextFingerprint = fingerprintNativeSkills(skillItems); + const nextCount = countNativeSkills(skillItems); + + setSkillSyncDetail( + t("instances.syncSkillsStatusAgent", { status: trackedCommand.status }), + ); + + if (nextFingerprint !== baselineFingerprint) { + fingerprintChanged = true; + resolvedSuccessDetail = + nextCount > baselineCount + ? t("instances.syncSkillsSuccessCount", { count: nextCount }) + : t("instances.syncSkillsSuccessUpdated"); + setSkillSyncPhase("success"); + setLastSkillSyncAt(new Date().toISOString()); + setSkillSyncDetail(resolvedSuccessDetail); + resolved = true; + break; + } + + if (isSkillSyncCommandFinished(trackedCommand.status)) { + if (trackedCommand.status === "succeeded") { + resolvedSuccessDetail = + countNativeSkills(latestSkills) > 0 + ? t("instances.syncSkillsSuccessCount", { + count: countNativeSkills(latestSkills), + }) + : t("instances.syncSkillsSuccessNoChange"); + setSkillSyncPhase("success"); + setLastSkillSyncAt(new Date().toISOString()); + setSkillSyncDetail(resolvedSuccessDetail); + } else { + syncFailed = true; + setSkillSyncPhase("failed"); + setSkillSyncDetail( + t("instances.syncSkillsCommandFailed", { + error: trackedCommand.error_message || trackedCommand.status, + }), + ); + } + resolved = true; + break; + } + + await sleep(SKILL_SYNC_POLL_MS); + } + + if (!resolved && skillSyncActiveRef.current) { + syncFailed = true; + setSkillSyncPhase("failed"); + setSkillSyncDetail(t("instances.syncSkillsTimeout")); + } + + if ( + usesWorkspaceSkillSync && + skillSyncActiveRef.current && + !syncFailed && + resolvedSuccessDetail + ) { + const hasPendingMaterialize = (skills: InstanceSkill[]) => + skills + .filter(isNativeInstanceSkill) + .some( + (item) => + isSkillPackagePending(item.skill) && + (item.skill?.package_materialize_status === "pending" || + item.skill?.package_materialize_status === "running" || + item.skill?.publish_blocked_reason === "skill_package_materializing" || + item.skill?.publish_blocked_reason === "skill_package_pending"), + ); + + let materializedSkills = latestSkills; + if (hasPendingMaterialize(materializedSkills)) { + setSkillSyncDetail(t("instances.syncSkillsMaterializing")); + const materializeDeadline = Date.now() + 30_000; + while (Date.now() < materializeDeadline && skillSyncActiveRef.current) { + await sleep(3_000); + materializedSkills = await skillService.listInstanceSkills(instanceId); + setInstanceSkills(materializedSkills); + if (!hasPendingMaterialize(materializedSkills)) { + break; + } + } + } + + const nextCount = countNativeSkills(materializedSkills); + setSkillSyncDetail( + nextCount > baselineCount + ? t("instances.syncSkillsSuccessCount", { count: nextCount }) + : fingerprintChanged + ? t("instances.syncSkillsSuccessUpdated") + : resolvedSuccessDetail, + ); + } + } catch (err: unknown) { + setSkillSyncPhase("failed"); + setSkillSyncDetail(hubErrorMessage(err, t("instances.syncSkillsFailed"))); + } finally { + skillSyncActiveRef.current = false; + } + }; + + const hubInstalledCount = useMemo( + () => hubCatalogRows.filter((row) => row.installed).length, + [hubCatalogRows], + ); + + const skillPanelSummary = t("instances.skillPanelSummary", { + native: countNativeSkills(instanceSkills), + installed: hubInstalledCount, + }); + + return ( + <> + } + summary={skillPanelSummary} + onExpandedChange={handlePanelExpandedChange} + headerActions={ + + } + > + + {t("skillHubPage.goToHub")} + + + {skillError ?
{skillError}
: null} + {(skillSyncPhase !== "idle" || lastSkillSyncAt) && ( +
+ {skillSyncDetail} + {lastSkillSyncAt && skillSyncPhase !== "syncing" ? ( +
+ {t("instances.syncSkillsLastSynced", { + time: formatDateTime(lastSkillSyncAt, locale), + })} +
+ ) : null} +
+ )} + +
+
+

{t("instances.nativeSkillsTitle")}

+

{t("instances.nativeSkillsDesc")}

+ setNativeSkillSearch(event.target.value)} + /> +
+ {instanceSkills.length === 0 ? ( +
+ {t("instances.noNativeSkills")} +
+ ) : filteredInstanceSkills.length === 0 ? ( +
+ {t("instances.noSkillsMatchingSearch")} +
+ ) : ( + instanceSkillsPagination.items.map((item) => ( + void handleImportSkillToLibrary(skillId)} + onRetryPackageCollect={(skillId) => void handleRetryPackageCollect(skillId)} + onPublish={(skillId) => { + setPublishSkillId(skillId); + setSelectedHubTagIds([]); + }} + onRemove={(skillId) => void handleRemoveSkill(skillId)} + shouldShowImportToLibrary={shouldShowImportToLibrary} + isSkillPackagePending={isSkillPackagePending} + isSkillPackageCollectFailed={isSkillPackageCollectFailed} + blockReasonLabel={blockReasonLabel} + isLiteInstance={isLiteInstance} + /> + )) + )} + {filteredInstanceSkills.length > INSTANCE_SKILL_PAGE_SIZE ? ( +
+

+ {t("instances.skillPageSummary", { + page: instanceSkillsPagination.currentPage, + totalPages: instanceSkillsPagination.totalPages, + totalSkills: filteredInstanceSkills.length, + })} +

+
+ + +
+
+ ) : null} +
+
+ +
+
+

{t("instances.hubCatalogTitle")}

+

{t("instances.hubCatalogDesc")}

+
+ setHubCatalogSearch(event.target.value)} + /> +
+ {hubCatalogRows.length === 0 ? ( +
+ {t("instances.noHubCatalogSkills")} +
+ ) : filteredHubCatalogRows.length === 0 ? ( +
+ {t("instances.noSkillsMatchingSearch")} +
+ ) : ( + filteredHubCatalogRows.map(({ skill, installed }) => ( +
+
+
+
+ {skill.name} + + {skillRiskLabel(t, skill.risk_level)} + + {installed ? ( + + {t("instances.hubSkillInstalled")} + + ) : null} + {(skill.tags || []).slice(0, 3).map((tag) => ( + + {hubTagLabel(tag)} + + ))} +
+

+ {skill.skill_key} + {skill.owner_username ? ` · ${skill.owner_username}` : ""} +

+
+
+ {installed ? ( + + ) : ( + + )} +
+
+
+ )) + )} +
+
+
+ + + {publishSkillId !== null ? ( +
+
+

{t("skillHubPage.publishFromInstance")}

+

{t("skillHubPage.selectTags")}

+
+ {hubTags + .filter((tag) => !tag.admin_only || user?.role === "admin") + .map((tag) => ( + + ))} +
+
+ + +
+
+
+ ) : null} + + ); +}; + +export default InstanceSkillHubPanel; diff --git a/frontend/src/components/SkillImportConflictDialog.tsx b/frontend/src/components/SkillImportConflictDialog.tsx new file mode 100644 index 0000000..23949b0 --- /dev/null +++ b/frontend/src/components/SkillImportConflictDialog.tsx @@ -0,0 +1,176 @@ +import React, { useEffect, useMemo, useState } from 'react'; +import { useI18n } from '../contexts/I18nContext'; +import type { SkillImportDecision, SkillImportPreviewItem } from '../types/skill'; + +type SkillImportConflictDialogProps = { + open: boolean; + items: SkillImportPreviewItem[]; + loading?: boolean; + onConfirm: (decisions: SkillImportDecision[]) => void; + onCancel: () => void; +}; + +const SkillImportConflictDialog: React.FC = ({ + open, + items, + loading = false, + onConfirm, + onCancel, +}) => { + const { t } = useI18n(); + const conflicts = useMemo( + () => items.filter((item) => item.conflict_type === 'content_changed'), + [items], + ); + const [choices, setChoices] = useState>({}); + + useEffect(() => { + if (!open) { + return; + } + const initial: Record = {}; + conflicts.forEach((item) => { + initial[item.directory_name] = 'new_version'; + }); + setChoices(initial); + }, [conflicts, open]); + + if (!open || conflicts.length === 0) { + return null; + } + + const single = conflicts.length === 1 ? conflicts[0] : null; + + const handleConfirm = () => { + const decisions: SkillImportDecision[] = items.map((item) => { + if (item.conflict_type === 'unchanged') { + return { directory_name: item.directory_name, action: 'skip' }; + } + if (item.conflict_type === 'none') { + return { directory_name: item.directory_name, action: 'new_version' }; + } + const action = choices[item.directory_name] || 'new_version'; + if (action === 'save_as_new') { + return { + directory_name: item.directory_name, + action, + skill_key: item.suggested_skill_key, + }; + } + return { directory_name: item.directory_name, action }; + }); + onConfirm(decisions); + }; + + return ( +
+
+

{t('skillHubPage.importConflict.title')}

+ + {single ? ( +

+ {t('skillHubPage.importConflict.message', { + name: single.existing_name || single.skill_key, + version: single.current_version_no ?? 1, + })} +

+ ) : ( +

{t('skillHubPage.importConflict.batchMessage')}

+ )} + + {single ? ( +
+ + + +
+ ) : ( + <> +
+ {conflicts.map((item) => ( +
+
{item.existing_name || item.skill_key}
+
+ {t('skillHubPage.importConflict.currentVersion', { + version: item.current_version_no ?? 1, + })} +
+ +
+ ))} +
+
+ + +
+ + )} +
+
+ ); +}; + +export default SkillImportConflictDialog; diff --git a/frontend/src/components/UserLayout.tsx b/frontend/src/components/UserLayout.tsx index 7ec4d85..8a2d4dc 100644 --- a/frontend/src/components/UserLayout.tsx +++ b/frontend/src/components/UserLayout.tsx @@ -5,6 +5,7 @@ import { Home, LogOut, Monitor, + Package, Settings, SlidersHorizontal, Users, @@ -17,6 +18,8 @@ interface UserLayoutProps { children: React.ReactNode; title?: string; titleAccessory?: React.ReactNode; + fillHeight?: boolean; + scrollableMain?: boolean; } interface UserNavItem { @@ -29,7 +32,13 @@ interface UserNavItem { const shellContainerClass = 'w-full px-3 sm:px-4 lg:px-5 2xl:px-6'; const appLogoSrc = '/lobster_logo.png'; -const UserLayout: React.FC = ({ children, title, titleAccessory }) => { +const UserLayout: React.FC = ({ + children, + title, + titleAccessory, + fillHeight = false, + scrollableMain = false, +}) => { const location = useLocation(); const navigate = useNavigate(); const { user, logout } = useAuth(); @@ -41,6 +50,7 @@ const UserLayout: React.FC = ({ children, title, titleAccessory { path: '/instances', label: t('nav.myInstances'), icon: Monitor }, { path: '/teams', label: 'Teams', icon: Users }, { path: '/openclaw-configs', label: t('nav.openClawConfigs'), icon: SlidersHorizontal }, + { path: '/skill-hub', label: t('nav.skillHub'), icon: Package }, { path: '/settings', label: t('nav.settings'), icon: Settings }, ]; @@ -116,7 +126,7 @@ const UserLayout: React.FC = ({ children, title, titleAccessory
{children}
-
+
-
+
{title && (
@@ -194,7 +204,15 @@ const UserLayout: React.FC = ({ children, title, titleAccessory
)} -
{children}
+
+ {children} +
diff --git a/frontend/src/components/WorkspaceFileManager.tsx b/frontend/src/components/WorkspaceFileManager.tsx index c288d54..224fdda 100644 --- a/frontend/src/components/WorkspaceFileManager.tsx +++ b/frontend/src/components/WorkspaceFileManager.tsx @@ -416,7 +416,7 @@ export function WorkspaceFileManager({ instanceId, initialPath, onMutation, refr const entries = entriesQuery.data ?? []; return ( -
+
diff --git a/frontend/src/lib/i18n.ts b/frontend/src/lib/i18n.ts index 96d75f2..0588b3f 100644 --- a/frontend/src/lib/i18n.ts +++ b/frontend/src/lib/i18n.ts @@ -1000,6 +1000,466 @@ const securityCenterTranslations: Record = { import { secplaneTranslations } from './secplane'; +const skillHubTranslations: Record = { + en: { + title: "Skill Hub", + subtitle: "Share and discover skills for your OpenClaw instances", + tabs: { catalog: "Browse", mine: "My Skills", admin: "Admin" }, + searchPlaceholder: "Search skills...", + allTags: "All tags", + upload: "Upload ZIP", + publish: "Publish to Hub", + unpublish: "Unpublish", + editTags: "Edit tags", + install: "Install to instance", + delete: "Delete", + download: "Download", + selectTags: "Select tags", + selectInstance: "Select instance", + noSkills: "No skills found", + visibility: { private: "Private", public: "Public" }, + publishable: "Ready to publish", + notPublishable: "Not ready to publish", + owner: "Owner", + instances: "Instances", + publishedAt: "Published", + sourceDeleted: "Source removed", + goToHub: "Browse Skill Hub", + publishFromInstance: "Publish to Hub", + importToLibrary: "Save to Skill Library", + importToLibraryPending: "Syncing skill package from instance. Please try again shortly.", + discoveredPublishHint: "For instance skills, retry package collect or upload a ZIP manually.", + importToLibrarySuccess: "Skill saved to your library.", + importToLibraryFailed: "Failed to save skill to library.", + alreadyInLibrary: "In Skill Library", + versionLabel: "v{version}", + configCenterHint: "Upload skill packages here, then publish them from Skill Hub.", + importConflict: { + title: "Duplicate skill detected", + message: "{name} already exists in your library (currently v{version}). The uploaded content is different.", + batchMessage: "Some skills in this archive already exist with different content. Choose how to handle each one.", + newVersion: "Save as new version", + saveAsNew: "Save as new skill ({key})", + skip: "Skip", + cancel: "Cancel", + confirmBatch: "Confirm import", + currentVersion: "Current version: v{version}", + }, + errors: { + load: "Failed to load Skill Hub", + publish: "Failed to publish skill", + install: "Failed to install skill", + delete: "Failed to delete skill", + upload: "Failed to upload skill", + updateTags: "Failed to update tags", + download: "Failed to download skill", + tagsRequired: "Select at least one tag", + }, + notices: { + published: "Skill published to hub", + unpublished: "Skill unpublished", + installed: "Skill installed to instance", + deleted: "Skill deleted", + uploaded: "Skill uploaded", + created: "Created {name}", + versioned: "Updated {name} to v{version}", + unchanged: "{name} is identical to the current version", + savedAsNew: "Saved as new skill: {key}", + tagsUpdated: "Tags updated", + }, + tags: { + productivity: "Productivity", + coding: "Coding", + browser: "Browser", + data: "Data", + communication: "Communication", + automation: "Automation", + research: "Research", + community: "Community", + "admin-curated": "Admin Curated", + featured: "Featured", + }, + blockReasons: { + skill_not_scanned: "Scan not completed", + skill_scan_failed: "Security scan failed", + skill_risk_blocked: "Risk level too high", + skill_package_pending: "Package not ready", + skill_package_materializing: "Packaging skill from workspace", + skill_package_materialize_failed: "Failed to package skill from workspace", + skill_package_collect_failed: "Failed to collect skill package from instance", + skill_package_md5_mismatch: "Skill package verification failed (MD5 mismatch, update Agent)", + skill_inactive: "Skill is inactive", + skill_deleted: "Skill was deleted", + skill_not_in_library: "Save to Skill Library first", + }, + }, + zh: { + title: "Skill Hub", + subtitle: "分享与发现适用于 OpenClaw 实例的 Skill", + tabs: { catalog: "浏览", mine: "我的 Skill", admin: "管理" }, + searchPlaceholder: "搜索 Skill...", + allTags: "全部标签", + upload: "上传 ZIP", + publish: "发布到 Hub", + unpublish: "下架", + editTags: "编辑标签", + install: "安装到实例", + delete: "删除", + download: "下载", + selectTags: "选择标签", + selectInstance: "选择实例", + noSkills: "暂无 Skill", + visibility: { private: "私有", public: "公开" }, + publishable: "可发布", + notPublishable: "暂不可发布", + owner: "所有者", + instances: "实例数", + publishedAt: "发布时间", + sourceDeleted: "源 Skill 已删除", + goToHub: "浏览 Skill Hub", + publishFromInstance: "发布到 Hub", + importToLibrary: "收录到 Skill 库", + importToLibraryPending: "正在从实例同步 Skill 包，请稍后重试。", + discoveredPublishHint: "实例内 skill 可重新采集包，或手动 ZIP 上传到 Skill Hub。", + importToLibrarySuccess: "已收录到 Skill 库。", + importToLibraryFailed: "收录到 Skill 库失败。", + alreadyInLibrary: "已在 Skill 库", + versionLabel: "v{version}", + configCenterHint: "可在此上传 Skill 包，然后前往 Skill Hub 发布。", + importConflict: { + title: "检测到同名 Skill", + message: "「{name}」已在你的库中（当前 v{version}），上传内容与现有版本不同。", + batchMessage: "压缩包中部分 Skill 与库中已有版本内容不同，请为每一项选择处理方式。", + newVersion: "作为新版本", + saveAsNew: "另存为新 Skill（{key}）", + skip: "跳过", + cancel: "取消", + confirmBatch: "确认导入", + currentVersion: "当前版本：v{version}", + }, + errors: { + load: "加载 Skill Hub 失败", + publish: "发布失败", + install: "安装失败", + delete: "删除失败", + upload: "上传失败", + updateTags: "更新标签失败", + download: "下载失败", + tagsRequired: "请至少选择一个标签", + }, + notices: { + published: "已发布到 Hub", + unpublished: "已下架", + installed: "已安装到实例", + deleted: "已删除", + uploaded: "上传成功", + created: "已新建 {name}", + versioned: "已升级 {name} → v{version}", + unchanged: "{name} 与现有版本一致，未产生变化", + savedAsNew: "已另存为新 Skill：{key}", + tagsUpdated: "标签已更新", + }, + tags: { + productivity: "效率", + coding: "编程", + browser: "浏览器", + data: "数据", + communication: "通信", + automation: "自动化", + research: "研究", + community: "社区", + "admin-curated": "管理员精选", + featured: "推荐", + }, + blockReasons: { + skill_not_scanned: "扫描未完成", + skill_scan_failed: "安全扫描失败", + skill_risk_blocked: "风险等级过高", + skill_package_pending: "安装包未就绪", + skill_package_materializing: "安装包打包中", + skill_package_materialize_failed: "安装包打包失败", + skill_package_collect_failed: "实例 Skill 包采集失败", + skill_package_md5_mismatch: "Skill 包校验失败（MD5 不一致，需更新 Agent）", + skill_inactive: "Skill 未激活", + skill_deleted: "Skill 已删除", + skill_not_in_library: "请先收录到 Skill 库", + }, + }, + ja: { + title: "Skill Hub", + subtitle: "OpenClaw インスタンス向け Skill の共有と探索", + tabs: { catalog: "閲覧", mine: "マイ Skill", admin: "管理" }, + searchPlaceholder: "Skill を検索...", + allTags: "すべてのタグ", + upload: "ZIP をアップロード", + publish: "Hub に公開", + unpublish: "公開解除", + editTags: "タグを編集", + install: "インスタンスにインストール", + delete: "削除", + download: "ダウンロード", + selectTags: "タグを選択", + selectInstance: "インスタンスを選択", + noSkills: "Skill がありません", + visibility: { private: "非公開", public: "公開" }, + publishable: "公開可能", + notPublishable: "公開不可", + owner: "所有者", + instances: "インスタンス数", + publishedAt: "公開日", + sourceDeleted: "ソース Skill 削除済み", + goToHub: "Skill Hub を見る", + publishFromInstance: "Hub に公開", + importToLibrary: "Skill ライブラリに保存", + importToLibraryPending: "インスタンスから Skill パッケージを同期中です。しばらくしてから再試行してください。", + importToLibrarySuccess: "Skill ライブラリに保存しました。", + importToLibraryFailed: "Skill ライブラリへの保存に失敗しました。", + alreadyInLibrary: "Skill ライブラリにあります", + versionLabel: "v{version}", + configCenterHint: "ここで Skill パッケージをアップロードし、Skill Hub から公開してください。", + importConflict: { + title: "同名 Skill を検出", + message: "「{name}」は既にライブラリにあります（現在 v{version}）。内容が異なります。", + batchMessage: "アーカイブ内の一部 Skill は既存版と内容が異なります。各項目の処理方法を選択してください。", + newVersion: "新バージョンとして保存", + saveAsNew: "新しい Skill として保存（{key}）", + skip: "スキップ", + cancel: "キャンセル", + confirmBatch: "インポートを確認", + currentVersion: "現在のバージョン: v{version}", + }, + errors: { + load: "Skill Hub の読み込みに失敗しました", + publish: "公開に失敗しました", + install: "インストールに失敗しました", + delete: "削除に失敗しました", + upload: "アップロードに失敗しました", + updateTags: "タグの更新に失敗しました", + download: "ダウンロードに失敗しました", + tagsRequired: "タグを1つ以上選択してください", + }, + notices: { + published: "Hub に公開しました", + unpublished: "公開を解除しました", + installed: "インスタンスにインストールしました", + deleted: "削除しました", + uploaded: "アップロードしました", + created: "{name} を新規作成しました", + versioned: "{name} を v{version} に更新しました", + unchanged: "{name} は既存バージョンと同一です", + savedAsNew: "新しい Skill として保存: {key}", + tagsUpdated: "タグを更新しました", + }, + tags: { + productivity: "生産性", + coding: "コーディング", + browser: "ブラウザ", + data: "データ", + communication: "通信", + automation: "自動化", + research: "リサーチ", + community: "コミュニティ", + "admin-curated": "管理者厳選", + featured: "おすすめ", + }, + blockReasons: { + skill_not_scanned: "スキャン未完了", + skill_scan_failed: "セキュリティスキャン失敗", + skill_risk_blocked: "リスクレベルが高すぎます", + skill_package_pending: "パッケージ未準備", + skill_package_materializing: "ワークスペースからパッケージ化中", + skill_package_materialize_failed: "ワークスペースからのパッケージ化に失敗", + skill_package_collect_failed: "インスタンスから Skill パッケージの取得に失敗", + skill_package_md5_mismatch: "Skill パッケージ検証失敗（MD5 不一致、Agent 更新が必要）", + skill_inactive: "Skill が非アクティブ", + skill_deleted: "Skill は削除済み", + skill_not_in_library: "先に Skill ライブラリに保存してください", + }, + }, + ko: { + title: "Skill Hub", + subtitle: "OpenClaw 인스턴스용 Skill 공유 및 탐색", + tabs: { catalog: "둘러보기", mine: "내 Skill", admin: "관리" }, + searchPlaceholder: "Skill 검색...", + allTags: "모든 태그", + upload: "ZIP 업로드", + publish: "Hub에 게시", + unpublish: "게시 취소", + editTags: "태그 편집", + install: "인스턴스에 설치", + delete: "삭제", + download: "다운로드", + selectTags: "태그 선택", + selectInstance: "인스턴스 선택", + noSkills: "Skill이 없습니다", + visibility: { private: "비공개", public: "공개" }, + publishable: "게시 가능", + notPublishable: "게시 불가", + owner: "소유자", + instances: "인스턴스 수", + publishedAt: "게시일", + sourceDeleted: "원본 Skill 삭제됨", + goToHub: "Skill Hub 보기", + publishFromInstance: "Hub에 게시", + importToLibrary: "Skill 라이브러리에 저장", + importToLibraryPending: "인스턴스에서 Skill 패키지를 동기화 중입니다. 잠시 후 다시 시도하세요.", + importToLibrarySuccess: "Skill 라이브러리에 저장되었습니다.", + importToLibraryFailed: "Skill 라이브러리 저장에 실패했습니다.", + alreadyInLibrary: "Skill 라이브러리에 있음", + versionLabel: "v{version}", + configCenterHint: "여기서 Skill 패키지를 업로드한 뒤 Skill Hub에서 게시하세요.", + importConflict: { + title: "동일 이름 Skill 감지", + message: "「{name}」이(가) 이미 라이브러리에 있습니다(현재 v{version}). 업로드 내용이 다릅니다.", + batchMessage: "아카이브의 일부 Skill 이 기존 버전과 내용이 다릅니다. 각 항목의 처리 방식을 선택하세요.", + newVersion: "새 버전으로 저장", + saveAsNew: "새 Skill 로 저장 ({key})", + skip: "건너뛰기", + cancel: "취소", + confirmBatch: "가져오기 확인", + currentVersion: "현재 버전: v{version}", + }, + errors: { + load: "Skill Hub 로드 실패", + publish: "게시 실패", + install: "설치 실패", + delete: "삭제 실패", + upload: "업로드 실패", + updateTags: "태그 업데이트 실패", + download: "다운로드 실패", + tagsRequired: "태그를 하나 이상 선택하세요", + }, + notices: { + published: "Hub에 게시됨", + unpublished: "게시 취소됨", + installed: "인스턴스에 설치됨", + deleted: "삭제됨", + uploaded: "업로드됨", + created: "{name} 생성됨", + versioned: "{name} → v{version} 업그레이드됨", + unchanged: "{name} 은(는) 기존 버전과 동일합니다", + savedAsNew: "새 Skill 로 저장: {key}", + tagsUpdated: "태그가 업데이트됨", + }, + tags: { + productivity: "생산성", + coding: "코딩", + browser: "브라우저", + data: "데이터", + communication: "커뮤니케이션", + automation: "자동화", + research: "리서치", + community: "커뮤니티", + "admin-curated": "관리자 큐레이션", + featured: "추천", + }, + blockReasons: { + skill_not_scanned: "스캔 미완료", + skill_scan_failed: "보안 스캔 실패", + skill_risk_blocked: "위험 수준이 너무 높음", + skill_package_pending: "패키지 준비 안 됨", + skill_package_materializing: "워크스페이스에서 패키지 생성 중", + skill_package_materialize_failed: "워크스페이스 패키지 생성 실패", + skill_package_collect_failed: "인스턴스 Skill 패키지 수집 실패", + skill_package_md5_mismatch: "Skill 패키지 검증 실패(MD5 불일치, Agent 업데이트 필요)", + skill_inactive: "Skill 비활성", + skill_deleted: "Skill 삭제됨", + skill_not_in_library: "먼저 Skill 라이브러리에 저장하세요", + }, + }, + de: { + title: "Skill Hub", + subtitle: "Skills für OpenClaw-Instanzen teilen und entdecken", + tabs: { catalog: "Durchsuchen", mine: "Meine Skills", admin: "Admin" }, + searchPlaceholder: "Skills suchen...", + allTags: "Alle Tags", + upload: "ZIP hochladen", + publish: "Im Hub veröffentlichen", + unpublish: "Veröffentlichung zurückziehen", + editTags: "Tags bearbeiten", + install: "Auf Instanz installieren", + delete: "Löschen", + download: "Herunterladen", + selectTags: "Tags auswählen", + selectInstance: "Instanz auswählen", + noSkills: "Keine Skills gefunden", + visibility: { private: "Privat", public: "Öffentlich" }, + publishable: "Veröffentlichbar", + notPublishable: "Nicht veröffentlichbar", + owner: "Besitzer", + instances: "Instanzen", + publishedAt: "Veröffentlicht", + sourceDeleted: "Quell-Skill entfernt", + goToHub: "Skill Hub öffnen", + publishFromInstance: "Im Hub veröffentlichen", + importToLibrary: "In Skill-Bibliothek speichern", + importToLibraryPending: "Skill-Paket wird von der Instanz synchronisiert. Bitte später erneut versuchen.", + importToLibrarySuccess: "In der Skill-Bibliothek gespeichert.", + importToLibraryFailed: "Speichern in der Skill-Bibliothek fehlgeschlagen.", + alreadyInLibrary: "In Skill-Bibliothek", + versionLabel: "v{version}", + configCenterHint: "Laden Sie Skill-Pakete hier hoch und veröffentlichen Sie sie im Skill Hub.", + importConflict: { + title: "Doppelter Skill erkannt", + message: "「{name}」 existiert bereits in Ihrer Bibliothek (aktuell v{version}). Der Inhalt unterscheidet sich.", + batchMessage: "Einige Skills im Archiv unterscheiden sich vom vorhandenen Inhalt. Wählen Sie pro Eintrag eine Aktion.", + newVersion: "Als neue Version speichern", + saveAsNew: "Als neuen Skill speichern ({key})", + skip: "Überspringen", + cancel: "Abbrechen", + confirmBatch: "Import bestätigen", + currentVersion: "Aktuelle Version: v{version}", + }, + errors: { + load: "Skill Hub konnte nicht geladen werden", + publish: "Veröffentlichung fehlgeschlagen", + install: "Installation fehlgeschlagen", + delete: "Löschen fehlgeschlagen", + upload: "Upload fehlgeschlagen", + updateTags: "Tags konnten nicht aktualisiert werden", + download: "Download fehlgeschlagen", + tagsRequired: "Mindestens ein Tag auswählen", + }, + notices: { + published: "Im Hub veröffentlicht", + unpublished: "Veröffentlichung zurückgezogen", + installed: "Auf Instanz installiert", + deleted: "Gelöscht", + uploaded: "Hochgeladen", + created: "{name} erstellt", + versioned: "{name} → v{version} aktualisiert", + unchanged: "{name} ist identisch mit der aktuellen Version", + savedAsNew: "Als neuen Skill gespeichert: {key}", + tagsUpdated: "Tags aktualisiert", + }, + tags: { + productivity: "Produktivität", + coding: "Coding", + browser: "Browser", + data: "Daten", + communication: "Kommunikation", + automation: "Automatisierung", + research: "Recherche", + community: "Community", + "admin-curated": "Admin-Kuratiert", + featured: "Empfohlen", + }, + blockReasons: { + skill_not_scanned: "Scan nicht abgeschlossen", + skill_scan_failed: "Sicherheitsscan fehlgeschlagen", + skill_risk_blocked: "Risikostufe zu hoch", + skill_package_pending: "Paket nicht bereit", + skill_package_materializing: "Skill wird aus Workspace gepackt", + skill_package_materialize_failed: "Skill-Paketierung aus Workspace fehlgeschlagen", + skill_package_collect_failed: "Skill-Paket konnte nicht von der Instanz gesammelt werden", + skill_package_md5_mismatch: "Skill-Paketprüfung fehlgeschlagen (MD5-Mismatch, Agent aktualisieren)", + skill_inactive: "Skill inaktiv", + skill_deleted: "Skill gelöscht", + skill_not_in_library: "Zuerst in Skill-Bibliothek speichern", + }, + }, +}; + export const translations: Record = { en: { app: { @@ -1054,12 +1514,14 @@ export const translations: Record = { aiGateway: "AI Gateway", aiAudit: "AI Audit", costs: "Costs", + sessionUsage: "Session Usage", riskRules: "Risk Rules", models: "Models", settings: "Settings", myInstances: "My Instances", portal: "Portal", openClawConfigs: "Resources", + skillHub: "Skill Hub", backToDashboard: "Back to Dashboard", backToUserDashboard: "Workspace", securityCenter: "Security Center", @@ -1069,6 +1531,7 @@ export const translations: Record = { adminLayout: adminLayoutTranslations.en, securityCenter: securityCenterTranslations.en, secplane: secplaneTranslations.en, + skillHubPage: skillHubTranslations.en, auth: { signInTitle: "Sign in to ClawManager", subtitle: "Virtual Desktop Management Platform", @@ -1221,6 +1684,8 @@ export const translations: Record = { "Inspect trace IDs, request and response payloads, risk hits, routing decisions, and invocation details.", costsDescription: "Review token usage, estimated spend, internal secure-model cost, and trace-linked billing details.", + sessionUsageDescription: + "Compare session-level token usage and estimated cost across all managed OpenClaw and Hermes instances.", riskRulesDescription: "Tune regex-based sensitive detection rules, severities, actions, and execution order.", }, @@ -1342,6 +1807,25 @@ export const translations: Record = { sevenDayView: "7-day view", noTrendData: "No trend data yet.", }, + sessionUsagePage: { + subtitle: + "Review session-level token usage and estimated cost across managed OpenClaw and Hermes instances.", + loadFailed: "Failed to load session usage overview", + exportFailed: "Failed to export session usage overview CSV", + loading: "Loading session usage overview...", + empty: "No managed runtime instances match the current filters.", + searchPlaceholder: "Search instance name, id, or type...", + instance: "Instance", + user: "User", + sessions: "Sessions", + totalTokens: "Total Tokens", + estimatedCost: "Estimated Cost", + instances: "Instances", + fallback: "Fallback Sessions", + fallbackYes: "{count} fallback", + fallbackNo: "None", + viewInstance: "Open instance", + }, modelManagementPage: { title: "Managed Models", subtitle: @@ -1912,6 +2396,48 @@ export const translations: Record = { "Restarting the instance. The desktop may disconnect briefly while the new stream settings apply.", restartSubmitted: "Restart requested. The desktop will reconnect when the instance is available again.", + governanceGatewayOk: "Gateway OK", + governanceExternalLLM: "External LLM config", + governanceSessionKeyMissing: "Session key missing", + sessionUsage: { + title: "Session Token Usage", + summary: "Total across all sessions", + panelSummary: "{{sessions}} sessions · {{tokens}} tokens · {{cost}}", + gatewayOnlyNotice: + "Only calls routed through the platform AI Gateway are included here.", + empty: "No gateway calls recorded yet.", + sessionKey: "Session", + invocations: "Calls", + lastActive: "Last active", + modelBreakdown: "By model", + recentTraces: "Recent traces", + fallbackWarning: + "Some sessions are missing stable session keys. Token totals may be split across pseudo sessions.", + loadFailed: "Failed to load session usage.", + detailLoadFailed: "Failed to load session detail.", + retry: "Retry", + prompt: "Prompt", + completion: "Completion", + total: "Total", + cost: "Cost", + sessions: "Sessions", + loading: "Loading session usage...", + detail: "Detail", + detailLoading: "Loading detail...", + detailEmpty: "No detail available.", + pageSummary: "Page {page} / {totalPages}", + prev: "Prev", + next: "Next", + timeRangeAll: "All time", + timeRange24h: "Last 24 hours", + timeRange7d: "Last 7 days", + timeRange30d: "Last 30 days", + autoRefresh: "Auto refresh (15s)", + refresh: "Refresh", + exportCsv: "Export CSV", + exportCsvLoading: "Exporting...", + exportFailed: "Failed to export session usage CSV.", + }, environmentVariables: "Environment Variables", clawManagerBuiltIns: "ClawManager Built-ins", hideBuiltIns: "Hide Built-ins", @@ -2014,15 +2540,55 @@ export const translations: Record = { infrastructureReady: "Infra ready", skillsSection: "Skills", skillManagement: "Skill Management", + panelExpand: "Expand", + panelCollapse: "Collapse", + skillPanelSummary: "{{native}} in instance · {{installed}} installed from Hub", selectSkill: "Select a skill", installingSkill: "Installing...", installSkill: "Install Skill", noSkillsReported: "No skills have been reported for this instance yet.", + skillsEmptyHint: "Skills shown inside the instance desktop are not synced automatically. Click Sync Skills to ask the agent to report the inventory.", + syncSkills: "Sync Skills", + syncSkillsInProgress: "Syncing…", + syncSkillsFailed: "Failed to request skill inventory sync.", + syncSkillsStatusRequesting: "Sending scan command to the agent…", + syncSkillsStatusAgent: "Agent is scanning skills ({{status}})…", + syncSkillsSuccessCount: "Sync complete. {{count}} instance skill(s) found.", + syncSkillsSuccessUpdated: "Sync complete. The skill list has been updated.", + syncSkillsSuccessNoChange: "Sync complete. No new skills were found.", + syncSkillsMaterializing: "Packaging discovered skills from workspace…", + syncSkillsTimeout: "Sync timed out. The agent did not respond in time. Check that the instance agent is online.", + syncSkillsCommandFailed: "Agent sync failed: {{error}}", + syncSkillsAgentOffline: "Instance agent is offline. Start the instance and wait for the agent to connect before syncing.", + syncSkillsLastSynced: "Last synced {{time}}", + skillPackageSyncing: "Package syncing", + skillPackageCollectFailed: "Package collection failed", + skillScanPending: "Awaiting security scan", + retryPackageCollect: "Retry package collect", + retryPackageMaterialize: "Retry packaging", + skillProvenanceNative: "Instance discovered", + skillProvenanceInjected: "Installed from Hub", + skillLibraryDiscovered: "Not in library", + skillLibraryUploaded: "In library", + nativeSkillsTitle: "Instance skills", + nativeSkillsDesc: "Skills discovered or created inside the instance workspace. Save them to your Skill library.", + hubCatalogTitle: "Skill Hub", + hubCatalogDesc: "Published skills from Skill Hub. Install them onto this instance with one click.", + hubSkillInstalled: "Installed", + hubSkillInstallAction: "Install to instance", + noHubCatalogSkills: "No installable skills in Skill Hub yet.", + installSkillRequiresAgent: "Instance agent is offline. Start the instance agent before installing skills.", + injectedSkillsTitle: "Installed from ClawManager", + injectedSkillsDesc: "Skills installed from Skill Hub or your library onto this instance.", + noNativeSkills: "No instance-discovered skills yet.", + noInjectedSkills: "No skills installed from ClawManager yet.", skillFallback: "Skill #{id}", lastSeenAt: "last seen {value}", removingSkill: "Removing...", removeSkill: "Remove", skillPagination: "Showing {from}-{to} of {total} skills", + skillSearchPlaceholder: "Search by name or key...", + noSkillsMatchingSearch: "No skills match your search.", previous: "Previous", nextPage: "Next", timelineSubtitle: "Instance operations and runtime history", @@ -2416,12 +2982,14 @@ export const translations: Record = { aiGateway: "AI 网关", aiAudit: "AI 审计", costs: "成本", + sessionUsage: "会话用量", riskRules: "风控规则", models: "模型", settings: "设置", myInstances: "我的实例", portal: "Portal", openClawConfigs: "资源管理", + skillHub: "Skill Hub", backToDashboard: "返回仪表盘", backToUserDashboard: "工作台", securityCenter: "安全中心", @@ -2431,6 +2999,7 @@ export const translations: Record = { adminLayout: adminLayoutTranslations.zh, securityCenter: securityCenterTranslations.zh, secplane: secplaneTranslations.zh, + skillHubPage: skillHubTranslations.zh, auth: { signInTitle: "登录 ClawManager", subtitle: "虚拟桌面管理平台", @@ -2578,6 +3147,8 @@ export const translations: Record = { "查看 trace、请求与响应负载、风险命中、路由决策和调用明细。", costsDescription: "查看 token 用量、预估费用、内部安全模型成本以及关联账单明细。", + sessionUsageDescription: + "跨 OpenClaw 与 Hermes 实例对比会话级 Token 用量与预估费用。", riskRulesDescription: "调整正则敏感规则、严重级别、动作和执行顺序。", }, aiAuditPage: { @@ -2693,6 +3264,24 @@ export const translations: Record = { sevenDayView: "7 天视图", noTrendData: "暂无趋势数据。", }, + sessionUsagePage: { + subtitle: "跨 OpenClaw 与 Hermes 实例查看会话级 Token 用量与预估费用。", + loadFailed: "加载会话用量总览失败", + exportFailed: "导出会话用量总览 CSV 失败", + loading: "正在加载会话用量总览...", + empty: "当前筛选条件下没有匹配的管理运行时实例。", + searchPlaceholder: "搜索实例名称、ID 或类型...", + instance: "实例", + user: "用户", + sessions: "会话数", + totalTokens: "Token 总量", + estimatedCost: "预估费用", + instances: "实例数", + fallback: "Fallback 会话", + fallbackYes: "{count} 个 fallback", + fallbackNo: "无", + viewInstance: "打开实例", + }, modelManagementPage: { title: "模型管理", subtitle: @@ -3222,6 +3811,47 @@ export const translations: Record = { restartInProgress: "正在重启实例，桌面会短暂断开，新桌面流设置会在重连后生效。", restartSubmitted: "已发起重启，实例可用后桌面会自动重新连接。", + governanceGatewayOk: "Gateway 正常", + governanceExternalLLM: "检测到外部 LLM 配置", + governanceSessionKeyMissing: "缺少 Session Key", + sessionUsage: { + title: "会话 Token 用量", + summary: "全部会话合计", + panelSummary: "{{sessions}} 个会话 · {{tokens}} tokens · {{cost}}", + gatewayOnlyNotice: "仅统计经平台 AI Gateway 的调用。", + empty: "暂无 Gateway 调用记录。", + sessionKey: "会话", + invocations: "调用次数", + lastActive: "最近活跃", + modelBreakdown: "按模型", + recentTraces: "最近 Trace", + fallbackWarning: + "部分会话缺少稳定的 session key，Token 可能被拆分到多个伪会话中。", + loadFailed: "加载会话用量失败。", + detailLoadFailed: "加载会话详情失败。", + retry: "重试", + prompt: "Prompt", + completion: "Completion", + total: "Total", + cost: "费用", + sessions: "会话数", + loading: "正在加载会话用量...", + detail: "详情", + detailLoading: "正在加载详情...", + detailEmpty: "暂无详情。", + pageSummary: "第 {page} / {totalPages} 页", + prev: "上一页", + next: "下一页", + timeRangeAll: "全部时间", + timeRange24h: "最近 24 小时", + timeRange7d: "最近 7 天", + timeRange30d: "最近 30 天", + autoRefresh: "自动刷新（15 秒）", + refresh: "刷新", + exportCsv: "导出 CSV", + exportCsvLoading: "导出中...", + exportFailed: "导出会话用量 CSV 失败。", + }, environmentVariables: "环境变量", clawManagerBuiltIns: "ClawManager 内置变量", hideBuiltIns: "收起内置变量", @@ -3312,15 +3942,55 @@ export const translations: Record = { infrastructureReady: "基础设施就绪", skillsSection: "技能", skillManagement: "技能管理", + panelExpand: "展开", + panelCollapse: "收起", + skillPanelSummary: "实例内 {{native}} 个 · Hub 已安装 {{installed}} 个", selectSkill: "选择技能", installingSkill: "安装中...", installSkill: "安装技能", noSkillsReported: "这个实例还没有上报任何技能。", + skillsEmptyHint: "实例桌面里看到的 Skill 不会自动出现在这里，需要 Agent 上报清单。可点击「同步 Skill」触发扫描。", + syncSkills: "同步 Skill", + syncSkillsInProgress: "同步中…", + syncSkillsFailed: "请求同步 Skill 清单失败。", + syncSkillsStatusRequesting: "正在向 Agent 发送扫描命令…", + syncSkillsStatusAgent: "Agent 正在扫描 skill 目录（{{status}}）…", + syncSkillsSuccessCount: "同步完成，共发现 {{count}} 个实例内 skill。", + syncSkillsSuccessUpdated: "同步完成，技能列表已更新。", + syncSkillsSuccessNoChange: "同步完成，未发现新的 skill。", + syncSkillsMaterializing: "正在从 workspace 打包 skill 安装包…", + syncSkillsTimeout: "同步超时：Agent 未在规定时间内响应，请确认实例 Agent 是否在线。", + syncSkillsCommandFailed: "Agent 执行失败：{{error}}", + syncSkillsAgentOffline: "实例 Agent 未连接，请确认实例已运行且 Agent 在线后再同步。", + syncSkillsLastSynced: "上次同步 {{time}}", + skillPackageSyncing: "包同步中", + skillPackageCollectFailed: "包采集失败", + skillScanPending: "等待安全扫描", + retryPackageCollect: "重新采集包", + retryPackageMaterialize: "重新打包", + skillProvenanceNative: "实例内发现", + skillProvenanceInjected: "从 Skill Hub 安装", + skillLibraryDiscovered: "未收录", + skillLibraryUploaded: "已在库", + nativeSkillsTitle: "实例内技能", + nativeSkillsDesc: "实例工作区中自行发现或创建的 skill，可收录到 Skill 库。", + hubCatalogTitle: "Skill Hub", + hubCatalogDesc: "Skill Hub 中已发布的 skill，可一键安装到本实例。", + hubSkillInstalled: "已安装", + hubSkillInstallAction: "安装到实例", + noHubCatalogSkills: "Skill Hub 暂无可安装的 skill。", + installSkillRequiresAgent: "实例 Agent 未连接，请先启动 Agent 后再安装 skill。", + injectedSkillsTitle: "从 ClawManager 安装", + injectedSkillsDesc: "从 Skill Hub 或库安装到本实例的 skill。", + noNativeSkills: "暂无实例内发现的 skill。", + noInjectedSkills: "暂未从 ClawManager 安装 skill。", skillFallback: "技能 #{id}", lastSeenAt: "最近出现于 {value}", removingSkill: "移除中...", removeSkill: "移除", skillPagination: "显示第 {from}-{to} 项，共 {total} 个技能", + skillSearchPlaceholder: "按名称或 key 搜索...", + noSkillsMatchingSearch: "没有匹配的 skill。", previous: "上一页", nextPage: "下一页", timelineSubtitle: "实例操作与运行时历史", @@ -3700,12 +4370,14 @@ export const translations: Record = { aiGateway: "AI Gateway", aiAudit: "AI 監査", costs: "コスト", + sessionUsage: "セッション使用量", riskRules: "リスクルール", models: "モデル", settings: "設定", myInstances: "マイインスタンス", portal: "ポータル", openClawConfigs: "リソース管理", + skillHub: "Skill Hub", backToDashboard: "ダッシュボードに戻る", backToUserDashboard: "ワークスペース", securityCenter: "セキュリティセンター", @@ -3715,6 +4387,7 @@ export const translations: Record = { adminLayout: adminLayoutTranslations.ja, securityCenter: securityCenterTranslations.ja, secplane: secplaneTranslations.ja, + skillHubPage: skillHubTranslations.ja, auth: { signInTitle: "ClawManager にサインイン", subtitle: "仮想デスクトップ管理プラットフォーム", @@ -3868,6 +4541,8 @@ export const translations: Record = { "Trace ID、リクエスト/レスポンス、リスクヒット、ルーティング判断、呼び出し詳細を確認します。", costsDescription: "Token 使用量、推定コスト、内部安全モデルコスト、関連する課金明細を確認します。", + sessionUsageDescription: + "管理対象の OpenClaw と Hermes インスタンス間で、セッション単位の Token 使用量と推定コストを比較します。", riskRulesDescription: "正規表現ベースの機密検知ルール、重大度、動作、実行順序を調整します。", }, @@ -3986,6 +4661,25 @@ export const translations: Record = { sevenDayView: "7 日表示", noTrendData: "まだ推移データがありません。", }, + sessionUsagePage: { + subtitle: + "管理対象の OpenClaw と Hermes インスタンス全体のセッション単位 Token 使用量と推定コストを確認します。", + loadFailed: "セッション使用量概要の読み込みに失敗しました", + exportFailed: "セッション使用量概要 CSV のエクスポートに失敗しました", + loading: "セッション使用量概要を読み込み中...", + empty: "現在のフィルターに一致する管理対象インスタンスがありません。", + searchPlaceholder: "インスタンス名、ID、タイプを検索...", + instance: "インスタンス", + user: "ユーザー", + sessions: "セッション", + totalTokens: "Token 合計", + estimatedCost: "推定コスト", + instances: "インスタンス数", + fallback: "Fallback セッション", + fallbackYes: "fallback {count} 件", + fallbackNo: "なし", + viewInstance: "インスタンスを開く", + }, modelManagementPage: { title: "モデル管理", subtitle: @@ -4496,6 +5190,48 @@ export const translations: Record = { "Restarting the instance. The desktop may disconnect briefly while the new stream settings apply.", restartSubmitted: "Restart requested. The desktop will reconnect when the instance is available again.", + governanceGatewayOk: "Gateway OK", + governanceExternalLLM: "外部 LLM 設定", + governanceSessionKeyMissing: "Session key 未設定", + sessionUsage: { + title: "セッション Token 使用量", + summary: "全セッション合計", + panelSummary: "{{sessions}} セッション · {{tokens}} tokens · {{cost}}", + gatewayOnlyNotice: + "プラットフォーム AI Gateway 経由の呼び出しのみ集計されます。", + empty: "Gateway 経由の呼び出しはまだ記録されていません。", + sessionKey: "セッション", + invocations: "呼び出し", + lastActive: "最終アクティブ", + modelBreakdown: "モデル別", + recentTraces: "最近の trace", + fallbackWarning: + "安定した session key がないセッションがあります。Token 合計が分散している可能性があります。", + loadFailed: "セッション使用量の読み込みに失敗しました。", + detailLoadFailed: "セッション詳細の読み込みに失敗しました。", + retry: "再試行", + prompt: "Prompt", + completion: "Completion", + total: "合計", + cost: "コスト", + sessions: "セッション", + loading: "セッション使用量を読み込み中...", + detail: "詳細", + detailLoading: "詳細を読み込み中...", + detailEmpty: "詳細がありません。", + pageSummary: "{page} / {totalPages} ページ", + prev: "前へ", + next: "次へ", + timeRangeAll: "全期間", + timeRange24h: "直近 24 時間", + timeRange7d: "直近 7 日", + timeRange30d: "直近 30 日", + autoRefresh: "自動更新（15 秒）", + refresh: "更新", + exportCsv: "CSV エクスポート", + exportCsvLoading: "エクスポート中...", + exportFailed: "セッション使用量 CSV のエクスポートに失敗しました。", + }, environmentVariables: "環境変数", clawManagerBuiltIns: "ClawManager 組み込み変数", hideBuiltIns: "組み込み変数を隠す", @@ -4599,15 +5335,54 @@ export const translations: Record = { infrastructureReady: "インフラ準備完了", skillsSection: "スキル", skillManagement: "スキル管理", + panelExpand: "展開", + panelCollapse: "閉じる", + skillPanelSummary: "インスタンス内 {{native}} 件 · Hub インストール {{installed}} 件", selectSkill: "スキルを選択", installingSkill: "適用中...", installSkill: "スキルを適用", noSkillsReported: "このインスタンスではまだスキルが報告されていません。", + skillsEmptyHint: "インスタンス画面内の Skill は自動同期されません。「Skill を同期」で Agent に inventory 報告を依頼してください。", + syncSkills: "Skill を同期", + syncSkillsInProgress: "同期中…", + syncSkillsFailed: "Skill inventory の同期要求に失敗しました。", + syncSkillsStatusRequesting: "Agent にスキャン命令を送信中…", + syncSkillsStatusAgent: "Agent が skill をスキャン中（{{status}}）…", + syncSkillsSuccessCount: "同期完了。インスタンス内 skill {{count}} 件。", + syncSkillsSuccessUpdated: "同期完了。スキル一覧を更新しました。", + syncSkillsSuccessNoChange: "同期完了。新しい skill は見つかりませんでした。", + syncSkillsMaterializing: "ワークスペースから skill パッケージを作成しています…", + syncSkillsTimeout: "同期がタイムアウトしました。Agent が応答していません。", + syncSkillsCommandFailed: "Agent の同期に失敗しました: {{error}}", + syncSkillsAgentOffline: "インスタンス Agent がオフラインです。Agent 接続後に再度お試しください。", + syncSkillsLastSynced: "最終同期 {{time}}", + skillPackageSyncing: "パッケージ同期中", + skillPackageCollectFailed: "パッケージ取得失敗", + retryPackageCollect: "パッケージ再取得", + retryPackageMaterialize: "再パッケージ化", + skillProvenanceNative: "インスタンス検出", + skillProvenanceInjected: "Hub からインストール", + skillLibraryDiscovered: "未保存", + skillLibraryUploaded: "ライブラリ内", + nativeSkillsTitle: "インスタンス内スキル", + nativeSkillsDesc: "インスタンス workspace で発見または作成された Skill。Skill ライブラリに保存できます。", + hubCatalogTitle: "Skill Hub", + hubCatalogDesc: "Skill Hub に公開された Skill を、このインスタンスへワンクリックでインストールできます。", + hubSkillInstalled: "インストール済み", + hubSkillInstallAction: "インスタンスへインストール", + noHubCatalogSkills: "Skill Hub にインストール可能な Skill がありません。", + installSkillRequiresAgent: "インスタンス Agent がオフラインです。Skill をインストールする前に Agent を起動してください。", + injectedSkillsTitle: "ClawManager からインストール", + injectedSkillsDesc: "Skill Hub またはライブラリからこのインスタンスにインストールされた Skill。", + noNativeSkills: "インスタンス内で検出された Skill はまだありません。", + noInjectedSkills: "ClawManager からインストールされた Skill はまだありません。", skillFallback: "スキル #{id}", lastSeenAt: "最終確認 {value}", removingSkill: "削除中...", removeSkill: "削除", skillPagination: "{total} 件中 {from}-{to} 件のスキルを表示", + skillSearchPlaceholder: "名前または key で検索...", + noSkillsMatchingSearch: "一致する Skill がありません。", previous: "前へ", nextPage: "次へ", timelineSubtitle: "インスタンス操作とランタイム履歴", @@ -5002,12 +5777,14 @@ export const translations: Record = { aiGateway: "AI Gateway", aiAudit: "AI 감사", costs: "비용", + sessionUsage: "세션 사용량", riskRules: "리스크 규칙", models: "모델", settings: "설정", myInstances: "내 인스턴스", portal: "포털", openClawConfigs: "리소스 관리", + skillHub: "Skill Hub", backToDashboard: "대시보드로 돌아가기", backToUserDashboard: "워크스페이스", securityCenter: "보안 센터", @@ -5017,6 +5794,7 @@ export const translations: Record = { adminLayout: adminLayoutTranslations.ko, securityCenter: securityCenterTranslations.ko, secplane: secplaneTranslations.ko, + skillHubPage: skillHubTranslations.ko, auth: { signInTitle: "ClawManager 로그인", subtitle: "가상 데스크톱 관리 플랫폼", @@ -5170,6 +5948,8 @@ export const translations: Record = { "Trace ID, 요청/응답 payload, 리스크 히트, 라우팅 결정, 호출 상세를 확인합니다.", costsDescription: "Token 사용량, 예상 비용, 내부 보안 모델 비용, 연계된 청구 세부를 확인합니다.", + sessionUsageDescription: + "관리 대상 OpenClaw 및 Hermes 인스턴스 전반의 세션 단위 Token 사용량과 예상 비용을 비교합니다.", riskRulesDescription: "정규식 기반 민감 규칙, 심각도, 동작, 실행 순서를 조정합니다.", }, @@ -5288,6 +6068,25 @@ export const translations: Record = { sevenDayView: "7일 보기", noTrendData: "추이 데이터가 아직 없습니다.", }, + sessionUsagePage: { + subtitle: + "관리 대상 OpenClaw 및 Hermes 인스턴스 전반의 세션 단위 Token 사용량과 예상 비용을 확인합니다.", + loadFailed: "세션 사용량 개요를 불러오지 못했습니다", + exportFailed: "세션 사용량 개요 CSV 내보내기에 실패했습니다", + loading: "세션 사용량 개요 불러오는 중...", + empty: "현재 필터와 일치하는 관리 대상 인스턴스가 없습니다.", + searchPlaceholder: "인스턴스 이름, ID, 유형 검색...", + instance: "인스턴스", + user: "사용자", + sessions: "세션", + totalTokens: "총 Token", + estimatedCost: "예상 비용", + instances: "인스턴스 수", + fallback: "Fallback 세션", + fallbackYes: "fallback {count}개", + fallbackNo: "없음", + viewInstance: "인스턴스 열기", + }, modelManagementPage: { title: "모델 관리", subtitle: @@ -5780,6 +6579,48 @@ export const translations: Record = { "Restarting the instance. The desktop may disconnect briefly while the new stream settings apply.", restartSubmitted: "Restart requested. The desktop will reconnect when the instance is available again.", + governanceGatewayOk: "Gateway OK", + governanceExternalLLM: "외부 LLM 설정", + governanceSessionKeyMissing: "Session key 누락", + sessionUsage: { + title: "세션 Token 사용량", + summary: "모든 세션 합계", + panelSummary: "{{sessions}}개 세션 · {{tokens}} tokens · {{cost}}", + gatewayOnlyNotice: + "플랫폼 AI Gateway를 경유한 호출만 집계됩니다.", + empty: "아직 Gateway 호출 기록이 없습니다.", + sessionKey: "세션", + invocations: "호출", + lastActive: "마지막 활동", + modelBreakdown: "모델별", + recentTraces: "최근 trace", + fallbackWarning: + "안정적인 session key가 없는 세션이 있습니다. Token 합계가 분산될 수 있습니다.", + loadFailed: "세션 사용량을 불러오지 못했습니다.", + detailLoadFailed: "세션 상세를 불러오지 못했습니다.", + retry: "다시 시도", + prompt: "Prompt", + completion: "Completion", + total: "합계", + cost: "비용", + sessions: "세션", + loading: "세션 사용량 불러오는 중...", + detail: "상세", + detailLoading: "상세 불러오는 중...", + detailEmpty: "상세 정보가 없습니다.", + pageSummary: "{page} / {totalPages} 페이지", + prev: "이전", + next: "다음", + timeRangeAll: "전체 기간", + timeRange24h: "최근 24시간", + timeRange7d: "최근 7일", + timeRange30d: "최근 30일", + autoRefresh: "자동 새로고침 (15초)", + refresh: "새로고침", + exportCsv: "CSV 내보내기", + exportCsvLoading: "내보내는 중...", + exportFailed: "세션 사용량 CSV 내보내기에 실패했습니다.", + }, environmentVariables: "환경 변수", clawManagerBuiltIns: "ClawManager 내장 변수", hideBuiltIns: "내장 변수 숨기기", @@ -5883,15 +6724,54 @@ export const translations: Record = { infrastructureReady: "인프라 준비 완료", skillsSection: "스킬", skillManagement: "스킬 관리", + panelExpand: "펼치기", + panelCollapse: "접기", + skillPanelSummary: "인스턴스 내 {{native}}개 · Hub 설치 {{installed}}개", selectSkill: "스킬 선택", installingSkill: "설치 중...", installSkill: "스킬 설치", noSkillsReported: "이 인스턴스에는 아직 보고된 스킬이 없습니다.", + skillsEmptyHint: "인스턴스 화면에 보이는 Skill 은 자동으로 여기에 표시되지 않습니다. 'Skill 동기화'로 Agent inventory 보고를 요청하세요.", + syncSkills: "Skill 동기화", + syncSkillsInProgress: "동기화 중…", + syncSkillsFailed: "Skill inventory 동기화 요청에 실패했습니다.", + syncSkillsStatusRequesting: "Agent에 스캔 명령 전송 중…", + syncSkillsStatusAgent: "Agent가 skill을 스캔 중({{status}})…", + syncSkillsSuccessCount: "동기화 완료. 인스턴스 skill {{count}}개.", + syncSkillsSuccessUpdated: "동기화 완료. skill 목록이 업데이트되었습니다.", + syncSkillsSuccessNoChange: "동기화 완료. 새 skill이 없습니다.", + syncSkillsMaterializing: "워크스페이스에서 skill 패키지를 생성하는 중…", + syncSkillsTimeout: "동기화 시간 초과. Agent가 응답하지 않았습니다.", + syncSkillsCommandFailed: "Agent 동기화 실패: {{error}}", + syncSkillsAgentOffline: "인스턴스 Agent가 오프라인입니다. Agent 연결 후 다시 시도하세요.", + syncSkillsLastSynced: "마지막 동기화 {{time}}", + skillPackageSyncing: "패키지 동기화 중", + skillPackageCollectFailed: "패키지 수집 실패", + retryPackageCollect: "패키지 재수집", + retryPackageMaterialize: "재패키징", + skillProvenanceNative: "인스턴스 감지", + skillProvenanceInjected: "Hub에서 설치", + skillLibraryDiscovered: "미저장", + skillLibraryUploaded: "라이브러리에 있음", + nativeSkillsTitle: "인스턴스 내 스킬", + nativeSkillsDesc: "인스턴스 workspace 에서 발견하거나 만든 Skill. Skill 라이브러리에 저장할 수 있습니다.", + hubCatalogTitle: "Skill Hub", + hubCatalogDesc: "Skill Hub 에 공개된 Skill 을 이 인스턴스에 원클릭으로 설치할 수 있습니다.", + hubSkillInstalled: "설치됨", + hubSkillInstallAction: "인스턴스에 설치", + noHubCatalogSkills: "Skill Hub 에 설치 가능한 Skill 이 없습니다.", + installSkillRequiresAgent: "인스턴스 Agent 가 오프라인입니다. Skill 설치 전 Agent 를 시작하세요.", + injectedSkillsTitle: "ClawManager에서 설치", + injectedSkillsDesc: "Skill Hub 또는 라이브러리에서 이 인스턴스에 설치된 Skill.", + noNativeSkills: "인스턴스에서 감지된 Skill 이 아직 없습니다.", + noInjectedSkills: "ClawManager에서 설치된 Skill 이 아직 없습니다.", skillFallback: "스킬 #{id}", lastSeenAt: "마지막 확인 {value}", removingSkill: "제거 중...", removeSkill: "제거", skillPagination: "총 {total}개 중 {from}-{to}개 스킬 표시", + skillSearchPlaceholder: "이름 또는 key 로 검색...", + noSkillsMatchingSearch: "검색 결과가 없습니다.", previous: "이전", nextPage: "다음", timelineSubtitle: "인스턴스 작업 및 런타임 기록", @@ -6285,12 +7165,14 @@ export const translations: Record = { aiGateway: "AI Gateway", aiAudit: "AI-Audit", costs: "Kosten", + sessionUsage: "Session-Nutzung", riskRules: "Risiko-Regeln", models: "Modelle", settings: "Einstellungen", myInstances: "Meine Instanzen", portal: "Portal", openClawConfigs: "Ressourcen", + skillHub: "Skill Hub", backToDashboard: "Zurück zum Dashboard", backToUserDashboard: "Arbeitsbereich", securityCenter: "Sicherheitszentrum", @@ -6300,6 +7182,7 @@ export const translations: Record = { adminLayout: adminLayoutTranslations.de, securityCenter: securityCenterTranslations.de, secplane: secplaneTranslations.de, + skillHubPage: skillHubTranslations.de, auth: { signInTitle: "Bei ClawManager anmelden", subtitle: "Plattform zur Verwaltung virtueller Desktops", @@ -6453,6 +7336,8 @@ export const translations: Record = { "Prüfen Sie Trace-IDs, Request-/Response-Payloads, Risikotreffer, Routing-Entscheidungen und Aufrufdetails.", costsDescription: "Prüfen Sie Token-Nutzung, geschätzte Kosten, interne Secure-Model-Kosten und zugehörige Abrechnungsdetails.", + sessionUsageDescription: + "Vergleichen Sie Token-Nutzung und geschätzte Kosten pro Session über verwaltete OpenClaw- und Hermes-Instanzen.", riskRulesDescription: "Passen Sie regex-basierte Erkennungsregeln, Schweregrade, Aktionen und Ausführungsreihenfolge an.", }, @@ -6575,6 +7460,25 @@ export const translations: Record = { sevenDayView: "7-Tage-Ansicht", noTrendData: "Noch keine Trenddaten.", }, + sessionUsagePage: { + subtitle: + "Prüfen Sie Token-Nutzung und geschätzte Kosten pro Session über verwaltete OpenClaw- und Hermes-Instanzen.", + loadFailed: "Session-Nutzungsübersicht konnte nicht geladen werden", + exportFailed: "Session-Nutzungsübersicht-CSV konnte nicht exportiert werden", + loading: "Session-Nutzungsübersicht wird geladen...", + empty: "Keine verwalteten Runtime-Instanzen entsprechen den aktuellen Filtern.", + searchPlaceholder: "Instanzname, ID oder Typ suchen...", + instance: "Instanz", + user: "Benutzer", + sessions: "Sessions", + totalTokens: "Token gesamt", + estimatedCost: "Geschätzte Kosten", + instances: "Instanzen", + fallback: "Fallback-Sessions", + fallbackYes: "{count} Fallback", + fallbackNo: "Keine", + viewInstance: "Instanz öffnen", + }, modelManagementPage: { title: "Modellverwaltung", subtitle: @@ -7089,6 +7993,48 @@ export const translations: Record = { "Restarting the instance. The desktop may disconnect briefly while the new stream settings apply.", restartSubmitted: "Restart requested. The desktop will reconnect when the instance is available again.", + governanceGatewayOk: "Gateway OK", + governanceExternalLLM: "Externe LLM-Konfiguration", + governanceSessionKeyMissing: "Session-Key fehlt", + sessionUsage: { + title: "Session-Token-Nutzung", + summary: "Summe aller Sessions", + panelSummary: "{{sessions}} Sessions · {{tokens}} Tokens · {{cost}}", + gatewayOnlyNotice: + "Es werden nur Aufrufe über das Plattform-AI-Gateway gezählt.", + empty: "Noch keine Gateway-Aufrufe erfasst.", + sessionKey: "Session", + invocations: "Aufrufe", + lastActive: "Zuletzt aktiv", + modelBreakdown: "Nach Modell", + recentTraces: "Letzte Traces", + fallbackWarning: + "Einige Sessions haben keinen stabilen Session-Key. Token-Summen können auf Pseudo-Sessions verteilt sein.", + loadFailed: "Session-Nutzung konnte nicht geladen werden.", + detailLoadFailed: "Session-Details konnten nicht geladen werden.", + retry: "Erneut versuchen", + prompt: "Prompt", + completion: "Completion", + total: "Gesamt", + cost: "Kosten", + sessions: "Sessions", + loading: "Session-Nutzung wird geladen...", + detail: "Details", + detailLoading: "Details werden geladen...", + detailEmpty: "Keine Details verfügbar.", + pageSummary: "Seite {page} / {totalPages}", + prev: "Zurück", + next: "Weiter", + timeRangeAll: "Gesamter Zeitraum", + timeRange24h: "Letzte 24 Stunden", + timeRange7d: "Letzte 7 Tage", + timeRange30d: "Letzte 30 Tage", + autoRefresh: "Auto-Aktualisierung (15 s)", + refresh: "Aktualisieren", + exportCsv: "CSV exportieren", + exportCsvLoading: "Export läuft...", + exportFailed: "Session-Nutzungs-CSV konnte nicht exportiert werden.", + }, environmentVariables: "Umgebungsvariablen", clawManagerBuiltIns: "ClawManager Built-ins", hideBuiltIns: "Built-ins ausblenden", @@ -7196,15 +8142,54 @@ export const translations: Record = { infrastructureReady: "Infrastruktur bereit", skillsSection: "Skills", skillManagement: "Skill-Verwaltung", + panelExpand: "Aufklappen", + panelCollapse: "Einklappen", + skillPanelSummary: "{{native}} in Instanz · {{installed}} aus Hub installiert", selectSkill: "Skill auswählen", installingSkill: "Wird installiert...", installSkill: "Skill installieren", noSkillsReported: "Für diese Instanz wurden noch keine Skills gemeldet.", + skillsEmptyHint: "Skills im Instanz-Desktop werden nicht automatisch hier angezeigt. Mit „Skills synchronisieren“ den Agent zum Inventory-Report auffordern.", + syncSkills: "Skills synchronisieren", + syncSkillsInProgress: "Synchronisierung…", + syncSkillsFailed: "Skill-Inventar-Sync konnte nicht angefordert werden.", + syncSkillsStatusRequesting: "Scan-Befehl wird an den Agent gesendet…", + syncSkillsStatusAgent: "Agent scannt Skills ({{status}})…", + syncSkillsSuccessCount: "Sync abgeschlossen. {{count}} Instanz-Skill(s) gefunden.", + syncSkillsSuccessUpdated: "Sync abgeschlossen. Skill-Liste wurde aktualisiert.", + syncSkillsSuccessNoChange: "Sync abgeschlossen. Keine neuen Skills gefunden.", + syncSkillsMaterializing: "Skill-Pakete werden aus dem Workspace erstellt…", + syncSkillsTimeout: "Sync-Zeitüberschreitung. Der Agent hat nicht rechtzeitig geantwortet.", + syncSkillsCommandFailed: "Agent-Sync fehlgeschlagen: {{error}}", + syncSkillsAgentOffline: "Instanz-Agent ist offline. Bitte warten, bis der Agent verbunden ist.", + syncSkillsLastSynced: "Zuletzt synchronisiert {{time}}", + skillPackageSyncing: "Paket wird synchronisiert", + skillPackageCollectFailed: "Paket-Sammlung fehlgeschlagen", + retryPackageCollect: "Paket erneut sammeln", + retryPackageMaterialize: "Erneut verpacken", + skillProvenanceNative: "In Instanz gefunden", + skillProvenanceInjected: "Vom Hub installiert", + skillLibraryDiscovered: "Nicht in Bibliothek", + skillLibraryUploaded: "In Bibliothek", + nativeSkillsTitle: "Instanz-Skills", + nativeSkillsDesc: "Im Instanz-Workspace entdeckte oder erstellte Skills. In der Skill-Bibliothek speichern.", + hubCatalogTitle: "Skill Hub", + hubCatalogDesc: "Veröffentlichte Skills aus dem Skill Hub mit einem Klick auf diese Instanz installieren.", + hubSkillInstalled: "Installiert", + hubSkillInstallAction: "Auf Instanz installieren", + noHubCatalogSkills: "Noch keine installierbaren Skills im Skill Hub.", + installSkillRequiresAgent: "Instanz-Agent ist offline. Bitte den Agent vor der Skill-Installation starten.", + injectedSkillsTitle: "Von ClawManager installiert", + injectedSkillsDesc: "Vom Skill Hub oder der Bibliothek auf diese Instanz installierte Skills.", + noNativeSkills: "Noch keine in der Instanz gefundenen Skills.", + noInjectedSkills: "Noch keine von ClawManager installierten Skills.", skillFallback: "Skill #{id}", lastSeenAt: "zuletzt gesehen {value}", removingSkill: "Wird entfernt...", removeSkill: "Entfernen", skillPagination: "Zeige Skill {from}-{to} von {total}", + skillSearchPlaceholder: "Nach Name oder Key suchen...", + noSkillsMatchingSearch: "Keine Skills entsprechen der Suche.", previous: "Zurück", nextPage: "Weiter", timelineSubtitle: "Instanzaktionen und Laufzeithistorie", diff --git a/frontend/src/pages/admin/AIGatewayPage.tsx b/frontend/src/pages/admin/AIGatewayPage.tsx index bdd65d8..5220879 100644 --- a/frontend/src/pages/admin/AIGatewayPage.tsx +++ b/frontend/src/pages/admin/AIGatewayPage.tsx @@ -25,6 +25,13 @@ const gatewaySections = [ accent: 'from-[#fff8ed] to-[#fffaf8]', icon: 'M12 8c-2.761 0-5 1.343-5 3s2.239 3 5 3 5 1.343 5 3-2.239 3-5 3m0-15c2.761 0 5 1.343 5 3m-5-3V4m0 16v-2m0-6c-2.761 0-5-1.343-5-3s2.239-3 5-3', }, + { + titleKey: 'nav.sessionUsage', + descriptionKey: 'aiGatewayPage.sessionUsageDescription', + path: '/admin/session-usage', + accent: 'from-[#eef8f3] to-[#fffaf8]', + icon: 'M4 19h16M7 16V8m5 8V5m5 11v-6', + }, { titleKey: 'nav.riskRules', descriptionKey: 'aiGatewayPage.riskRulesDescription', diff --git a/frontend/src/pages/admin/SessionUsageOverviewPage.tsx b/frontend/src/pages/admin/SessionUsageOverviewPage.tsx new file mode 100644 index 0000000..895ed0f --- /dev/null +++ b/frontend/src/pages/admin/SessionUsageOverviewPage.tsx @@ -0,0 +1,314 @@ +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { Link } from "react-router-dom"; +import AdminLayout from "../../components/AdminLayout"; +import { useI18n } from "../../contexts/I18nContext"; +import { + adminService, + type SessionUsageOverview, + type SessionUsageOverviewItem, +} from "../../services/adminService"; +import { + downloadSessionUsageCsv, + resolveSessionUsageSince, + type SessionUsageTimeRange, +} from "../../utils/sessionUsageExport"; + +const PAGE_SIZE = 20; +const AUTO_REFRESH_MS = 15000; + +function formatNumber(value: number) { + return new Intl.NumberFormat().format(value); +} + +function formatCost(value: number, currency: string) { + return new Intl.NumberFormat(undefined, { + style: "currency", + currency: currency || "USD", + maximumFractionDigits: 4, + }).format(value); +} + +const SessionUsageOverviewPage: React.FC = () => { + const { t } = useI18n(); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(""); + const [search, setSearch] = useState(""); + const [debouncedSearch, setDebouncedSearch] = useState(""); + const [page, setPage] = useState(1); + const [timeRange, setTimeRange] = useState("all"); + const [autoRefresh, setAutoRefresh] = useState(false); + const [exporting, setExporting] = useState(false); + const [overview, setOverview] = useState(null); + + useEffect(() => { + const timer = window.setTimeout(() => setDebouncedSearch(search.trim()), 300); + return () => window.clearTimeout(timer); + }, [search]); + + const since = useMemo(() => resolveSessionUsageSince(timeRange), [timeRange]); + + const loadOverview = useCallback(async (options?: { silent?: boolean }) => { + if (!options?.silent) { + setLoading(true); + } + setError(""); + try { + const data = await adminService.getSessionUsageOverview({ + page, + limit: PAGE_SIZE, + search: debouncedSearch || undefined, + since, + }); + setOverview(data); + } catch (err: unknown) { + setError(err instanceof Error ? err.message : t("sessionUsagePage.loadFailed")); + } finally { + if (!options?.silent) { + setLoading(false); + } + } + }, [debouncedSearch, page, since, t]); + + useEffect(() => { + void loadOverview(); + }, [loadOverview]); + + useEffect(() => { + setPage(1); + }, [debouncedSearch, timeRange]); + + useEffect(() => { + if (!autoRefresh) { + return; + } + const timer = window.setInterval(() => { + void loadOverview({ silent: true }); + }, AUTO_REFRESH_MS); + return () => window.clearInterval(timer); + }, [autoRefresh, loadOverview]); + + const totalPages = useMemo(() => { + if (!overview) { + return 1; + } + return Math.max(1, Math.ceil(overview.total / PAGE_SIZE)); + }, [overview]); + + const exportCsv = async () => { + setExporting(true); + try { + const rows: SessionUsageOverviewItem[] = []; + let pageCursor = 1; + let total = 0; + do { + const batch = await adminService.getSessionUsageOverview({ + page: pageCursor, + limit: 100, + search: debouncedSearch || undefined, + since, + }); + rows.push(...batch.items); + total = batch.total; + pageCursor += 1; + } while (rows.length < total); + + const header = [ + "instance_id", + "instance_name", + "instance_type", + "user_id", + "session_count", + "total_tokens", + "estimated_cost", + "currency", + "fallback_sessions", + ]; + const csvRows = rows.map((item) => + [ + item.instance_id, + item.instance_name, + item.instance_type, + item.user_id, + item.summary.session_count, + item.summary.total_tokens, + item.summary.total_estimated_cost, + item.summary.currency, + item.compliance.fallback_session_count, + ] + .map((cell) => { + const value = String(cell ?? ""); + return /[",\n]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value; + }) + .join(","), + ); + downloadSessionUsageCsv([header.join(","), ...csvRows].join("\n"), "session-usage-overview.csv"); + } catch (err: unknown) { + setError(err instanceof Error ? err.message : t("sessionUsagePage.exportFailed")); + } finally { + setExporting(false); + } + }; + + return ( + +
+
+

{t("sessionUsagePage.subtitle")}

+
+ setSearch(event.target.value)} + placeholder={t("sessionUsagePage.searchPlaceholder")} + className="min-w-[220px] flex-1 rounded-md border border-[#ead8cf] px-3 py-2 text-sm" + /> + + + + +
+
+ + {error && ( +
+
{error}
+
+ )} + + {loading || !overview ? ( +
+ {t("sessionUsagePage.loading")} +
+ ) : ( + <> +
+ + + + +
+ +
+ {overview.items.length === 0 ? ( +
{t("sessionUsagePage.empty")}
+ ) : ( + + + + + + + + + + + + + {overview.items.map((item) => ( + + + + + + + + + + ))} + +
{t("sessionUsagePage.instance")}{t("sessionUsagePage.user")}{t("sessionUsagePage.sessions")}{t("sessionUsagePage.totalTokens")}{t("sessionUsagePage.estimatedCost")}{t("sessionUsagePage.fallback")} +
+
{item.instance_name}
+
+ #{item.instance_id} · {item.instance_type} +
+ 		{item.user_id}{formatNumber(item.summary.session_count)}{formatNumber(item.summary.total_tokens)} + {formatCost(item.summary.total_estimated_cost, item.summary.currency)} + + {item.compliance.has_fallback_sessions + ? t("sessionUsagePage.fallbackYes", { + count: item.compliance.fallback_session_count, + }) + : t("sessionUsagePage.fallbackNo")} + + + {t("sessionUsagePage.viewInstance")} + +
+ )} +
+ + {overview.total > PAGE_SIZE && ( +
+ {t("instances.sessionUsage.pageSummary", { page, totalPages })} +
+ + +
+
+ )} + + )} +
+ + ); +}; + +function MetricCard({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +export default SessionUsageOverviewPage; diff --git a/frontend/src/pages/instances/CreateInstancePage.tsx b/frontend/src/pages/instances/CreateInstancePage.tsx index e3be758..2212727 100644 --- a/frontend/src/pages/instances/CreateInstancePage.tsx +++ b/frontend/src/pages/instances/CreateInstancePage.tsx @@ -7,7 +7,7 @@ import OpenClawConfigPlanSection, { import UserLayout from "../../components/UserLayout"; import { useAuth } from "../../contexts/AuthContext"; import { instanceService } from "../../services/instanceService"; -import { skillService } from "../../services/skillService"; +import { skillHubService } from "../../services/skillHubService"; import { userService } from "../../services/userService"; import { INSTANCE_TYPES, PRESET_CONFIGS } from "../../types/instance"; import type { CreateInstanceRequest, InstanceMode } from "../../types/instance"; @@ -612,10 +612,12 @@ const CreateInstancePage: React.FC = () => { const loadSkills = async () => { try { setSkillLoading(true); - const items = await skillService.listSkills(); + const items = await skillHubService.listAttachable(); const activeSkills = items.filter((item) => item.status === "active"); const attachableSkills = activeSkills.filter( - (item) => !["medium", "high"].includes(item.risk_level.trim().toLowerCase()), + (item) => + item.risk_level !== "medium" && + item.risk_level !== "high", ); setAvailableSkills(attachableSkills); setSkillInventorySummary({ diff --git a/frontend/src/pages/instances/InstanceDetailPage.tsx b/frontend/src/pages/instances/InstanceDetailPage.tsx index c14956e..5a5c677 100644 --- a/frontend/src/pages/instances/InstanceDetailPage.tsx +++ b/frontend/src/pages/instances/InstanceDetailPage.tsx @@ -1,11 +1,10 @@ -import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; +import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { Link, useNavigate, useParams } from "react-router-dom"; import { ArrowLeft, Ban, BarChart3, Clock3, - ChevronDown, Copy, Cpu, Eye, @@ -16,14 +15,14 @@ import { MemoryStick, Network, Play, - Plus, RotateCw, Square, Trash2, - Upload, X, } from "lucide-react"; import ConfirmDialog from "../../components/ConfirmDialog"; +import InstanceSkillHubPanel from "../../components/InstanceSkillHubPanel"; +import InstanceSessionUsagePanel from "../../components/InstanceSessionUsagePanel"; import { InstanceServiceFrame } from "../../components/InstanceServiceFrame"; import UserLayout from "../../components/UserLayout"; import { WorkspaceFileManager } from "../../components/WorkspaceFileManager"; @@ -31,7 +30,6 @@ import { useI18n } from "../../contexts/I18nContext"; import { useInstanceStatusWebSocket } from "../../hooks/useWebSocket"; import type { InstanceStatusUpdate } from "../../hooks/useWebSocket"; import { instanceService } from "../../services/instanceService"; -import { skillService } from "../../services/skillService"; import type { ExternalAccessExpirationMode, ExternalAccessExpirationPreset, @@ -44,7 +42,6 @@ import type { InstanceStatus, DesktopStreamProfile, } from "../../types/instance"; -import type { InstanceSkill, Skill } from "../../types/skill"; const META_POLL_INTERVAL_MS = 5000; const RUNTIME_POLL_INTERVAL_MS = 5000; @@ -112,25 +109,6 @@ function supportsWorkspace(instance: Instance) { return instance.type === "openclaw" || instance.type === "hermes" || Boolean(instance.workspace_path); } -function workspaceInitialPath(instance: Instance, isDedicated: boolean) { - const type = instance.type.trim().toLowerCase(); - if (type === "hermes") { - return isDedicated ? ".hermes" : "home/.hermes"; - } - if (type === "openclaw" && !isDedicated) { - return "home/.openclaw"; - } - return isDedicated ? "/config" : undefined; -} - -function isUsableSkillStatus(skill: Skill) { - const status = skill.status.trim().toLowerCase(); - return status === "" || status === "active" || status === "enabled"; -} -function isAttachedInstanceSkill(item: InstanceSkill) { - return item.status.trim().toLowerCase() !== "removed" && !item.removed_at; -} - function getErrorMessage(err: unknown, fallback: string) { const responseError = (err as { response?: { data?: { error?: string } } })?.response?.data?.error; if (responseError) { @@ -330,25 +308,16 @@ const InstanceDetailPage: React.FC = () => { const [externalCustomExpiresAt, setExternalCustomExpiresAt] = useState(""); const [runtimeDetails, setRuntimeDetails] = useState(null); const [runtimeError, setRuntimeError] = useState(null); - const [instanceSkills, setInstanceSkills] = useState([]); - const [availableSkills, setAvailableSkills] = useState([]); - const [skillInventorySummary, setSkillInventorySummary] = useState({ total: 0, hiddenByRisk: 0 }); - const [selectedSkillIds, setSelectedSkillIds] = useState([]); - const [skillPickerOpen, setSkillPickerOpen] = useState(false); - const skillPickerRef = useRef(null); - const skillUploadInputRef = useRef(null); - const desktopFrameRef = useRef(null); - const [skillLoading, setSkillLoading] = useState(false); - const [skillError, setSkillError] = useState(null); - const [skillNotice, setSkillNotice] = useState(null); - const [workspaceRefreshKey, setWorkspaceRefreshKey] = useState(0); - const [desktopFrameHeight, setDesktopFrameHeight] = useState(null); const [desktopStreamProfile, setDesktopStreamProfile] = useState("standard"); const [desktopStreamSavedProfile, setDesktopStreamSavedProfile] = useState(""); const [desktopStreamMessage, setDesktopStreamMessage] = useState(null); const [actionMessage, setActionMessage] = useState(null); + const [skillPanelExpanded, setSkillPanelExpanded] = useState(false); + const [sessionPanelExpanded, setSessionPanelExpanded] = useState(false); + const [workspaceHeightPx, setWorkspaceHeightPx] = useState(null); + const workspaceSectionRef = useRef(null); const fetchMeta = useCallback( async (targetInstanceId: number, options?: { background?: boolean }) => { @@ -398,44 +367,6 @@ const InstanceDetailPage: React.FC = () => { } }, []); - const fetchSkills = useCallback(async (targetInstanceId: number) => { - try { - setSkillLoading(true); - const attached = await skillService.listInstanceSkills(targetInstanceId); - let reusable: Skill[] = []; - try { - reusable = await skillService.listAvailableInstanceSkills(targetInstanceId); - } catch { - reusable = []; - } - if (reusable.length === 0) { - reusable = await skillService.listSkills(); - } - const activeAttached = attached.filter(isAttachedInstanceSkill); - const attachedIds = new Set(activeAttached.map((item) => item.skill_id)); - const activeReusableSkills = reusable.filter(isUsableSkillStatus); - const riskAllowedSkills = activeReusableSkills.filter( - (skill) => !["medium", "high"].includes(skill.risk_level.trim().toLowerCase()), - ); - setInstanceSkills(activeAttached); - setSkillInventorySummary({ - total: activeReusableSkills.length, - hiddenByRisk: activeReusableSkills.length - riskAllowedSkills.length, - }); - const attachableSkills = riskAllowedSkills.filter((skill) => !attachedIds.has(skill.id)); - const attachableSkillIds = new Set(attachableSkills.map((skill) => skill.id)); - setAvailableSkills(attachableSkills); - setSelectedSkillIds((current) => current.filter((skillId) => attachableSkillIds.has(skillId))); - if (attachableSkills.length === 0) { - setSkillPickerOpen(false); - } - setSkillError(null); - } catch (err: unknown) { - setSkillError(getErrorMessage(err, "Failed to load skills")); - } finally { - setSkillLoading(false); - } - }, []); useEffect(() => { const savedProfile = instance?.desktop_stream_profile || ""; setDesktopStreamProfile(savedProfile || "standard"); @@ -495,61 +426,14 @@ const InstanceDetailPage: React.FC = () => { ); useEffect(() => { - if (!skillPickerOpen) { - return undefined; - } - const handlePointerDown = (event: MouseEvent) => { - if (!skillPickerRef.current?.contains(event.target as Node)) { - setSkillPickerOpen(false); - } - }; - document.addEventListener("mousedown", handlePointerDown); - return () => document.removeEventListener("mousedown", handlePointerDown); - }, [skillPickerOpen]); - - useLayoutEffect(() => { - const element = desktopFrameRef.current; - if (!element) { - setDesktopFrameHeight(null); - return undefined; - } - - const updateDesktopFrameHeight = () => { - const nextHeight = Math.round(element.getBoundingClientRect().height); - setDesktopFrameHeight((current) => (nextHeight > 0 && current !== nextHeight ? nextHeight : current)); - }; - - updateDesktopFrameHeight(); - - if (typeof ResizeObserver === "undefined") { - window.addEventListener("resize", updateDesktopFrameHeight); - return () => window.removeEventListener("resize", updateDesktopFrameHeight); - } - - const observer = new ResizeObserver(updateDesktopFrameHeight); - observer.observe(element); - return () => observer.disconnect(); - }, [instance?.id, isDedicatedInstance]); - useEffect(() => { - if (!instanceId || Number.isNaN(instanceId)) { + if (!instanceId || Number.isNaN(instanceId) || !isDedicatedInstance) { setRuntimeDetails(null); setRuntimeError(null); - setInstanceSkills([]); - setAvailableSkills([]); - setSkillInventorySummary({ total: 0, hiddenByRisk: 0 }); - setSelectedSkillIds([]); - setSkillPickerOpen(false); - setSkillNotice(null); return; } - void fetchSkills(instanceId); - if (isDedicatedInstance) { - void fetchRuntimeDetails(instanceId); - } else { - setRuntimeDetails(null); - setRuntimeError(null); - } - }, [fetchRuntimeDetails, fetchSkills, instanceId, isDedicatedInstance]); + void fetchRuntimeDetails(instanceId); + }, [fetchRuntimeDetails, instanceId, isDedicatedInstance]); + useEffect(() => { if (!instanceId || Number.isNaN(instanceId) || !isDedicatedInstance) { return; @@ -563,6 +447,72 @@ const InstanceDetailPage: React.FC = () => { return () => window.clearInterval(timer); }, [fetchRuntimeDetails, instanceId, isDedicatedInstance]); + useEffect(() => { + if (isDedicatedInstance) { + return; + } + const section = workspaceSectionRef.current; + if (!section) { + return; + } + + const syncWorkspaceHeight = () => { + if (skillPanelExpanded || sessionPanelExpanded) { + return; + } + const nextHeight = section.getBoundingClientRect().height; + if (nextHeight > 0) { + setWorkspaceHeightPx(nextHeight); + } + }; + + syncWorkspaceHeight(); + const observer = new ResizeObserver(syncWorkspaceHeight); + observer.observe(section); + window.addEventListener("resize", syncWorkspaceHeight); + + return () => { + observer.disconnect(); + window.removeEventListener("resize", syncWorkspaceHeight); + }; + }, [isDedicatedInstance, sessionPanelExpanded, skillPanelExpanded]); + + const bottomPanelExpanded = skillPanelExpanded || sessionPanelExpanded; + + useLayoutEffect(() => { + if (isDedicatedInstance || !bottomPanelExpanded || workspaceHeightPx !== null) { + return; + } + const section = workspaceSectionRef.current; + if (!section) { + return; + } + const nextHeight = section.getBoundingClientRect().height; + if (nextHeight > 0) { + setWorkspaceHeightPx(nextHeight); + } + }, [bottomPanelExpanded, isDedicatedInstance, workspaceHeightPx]); + + const handleSkillPanelExpandedChange = useCallback((expanded: boolean) => { + if (expanded && workspaceSectionRef.current) { + const nextHeight = workspaceSectionRef.current.getBoundingClientRect().height; + if (nextHeight > 0) { + setWorkspaceHeightPx(nextHeight); + } + } + setSkillPanelExpanded(expanded); + }, []); + + const handleSessionPanelExpandedChange = useCallback((expanded: boolean) => { + if (expanded && workspaceSectionRef.current) { + const nextHeight = workspaceSectionRef.current.getBoundingClientRect().height; + if (nextHeight > 0) { + setWorkspaceHeightPx(nextHeight); + } + } + setSessionPanelExpanded(expanded); + }, []); + const availability = useMemo(() => { if (status?.availability) { return status.availability; @@ -745,110 +695,6 @@ const InstanceDetailPage: React.FC = () => { } }; - const attachSelectedSkills = async () => { - if (!instance || selectedSkillIds.length === 0) { - return; - } - const skillOptionIds = new Set(skillOptions.map((skill) => skill.id)); - const skillIds = selectedSkillIds.filter((skillId) => skillOptionIds.has(skillId)); - if (skillIds.length === 0) { - setSelectedSkillIds([]); - return; - } - try { - setSkillLoading(true); - setSkillError(null); - await Promise.all(skillIds.map((skillId) => skillService.attachSkillToInstance(instance.id, skillId))); - setSelectedSkillIds([]); - setSkillPickerOpen(false); - setSkillNotice(`Attached ${skillIds.length} skill${skillIds.length === 1 ? "" : "s"}.`); - await fetchSkills(instance.id); - refreshWorkspaceFiles(); - } catch (err: unknown) { - setSkillError(getErrorMessage(err, "Failed to attach skill")); - } finally { - setSkillLoading(false); - } - }; - - const refreshWorkspaceFiles = () => { - setWorkspaceRefreshKey((current) => current + 1); - }; - - const handleSkillArchiveUpload = async (fileList?: FileList | null) => { - const file = fileList?.[0]; - if (skillUploadInputRef.current) { - skillUploadInputRef.current.value = ""; - } - if (!instance || !file) { - return; - } - if (!file.name.toLowerCase().endsWith(".zip")) { - setSkillError("Only .zip skill archives are supported."); - setSkillNotice(null); - return; - } - - try { - setSkillLoading(true); - setSkillError(null); - setSkillNotice(null); - const importedSkills = await skillService.importSkills(file); - let attachedCount = 0; - const failures: string[] = []; - for (const importedSkill of importedSkills) { - try { - await skillService.attachSkillToInstance(instance.id, importedSkill.id); - attachedCount += 1; - } catch (err: unknown) { - failures.push(`${importedSkill.name || importedSkill.skill_key}: ${getErrorMessage(err, "attach failed")}`); - } - } - setSelectedSkillIds([]); - setSkillPickerOpen(false); - await fetchSkills(instance.id); - refreshWorkspaceFiles(); - const importedCount = importedSkills.length; - if (failures.length > 0) { - setSkillError(failures.join("; ")); - } - if (attachedCount > 0) { - setSkillNotice(`Imported ${importedCount} skill${importedCount === 1 ? "" : "s"}; attached ${attachedCount}.`); - } else { - setSkillNotice(`Imported ${importedCount} skill${importedCount === 1 ? "" : "s"}; none attached.`); - } - } catch (err: unknown) { - setSkillError(getErrorMessage(err, "Failed to upload skill archive")); - setSkillNotice(null); - } finally { - setSkillLoading(false); - } - }; - const toggleSelectedSkill = (skillId: number) => { - setSelectedSkillIds((current) => - current.includes(skillId) ? current.filter((item) => item !== skillId) : [...current, skillId], - ); - }; - - const removeInstanceSkill = async (skillId: number) => { - if (!instance) { - return; - } - try { - setSkillLoading(true); - setSkillError(null); - await skillService.removeSkillFromInstance(instance.id, skillId); - setInstanceSkills((current) => current.filter((item) => item.skill_id !== skillId)); - setSkillNotice("Skill removed."); - await fetchSkills(instance.id); - refreshWorkspaceFiles(); - } catch (err: unknown) { - setSkillError(getErrorMessage(err, "Failed to remove skill")); - } finally { - setSkillLoading(false); - } - }; - if (loading) { return ( @@ -1157,128 +1003,29 @@ const InstanceDetailPage: React.FC = () => {
); - const renderSkillPanel = () => ( -
-
-
-
- -

Instance Skills

-
-
{instanceSkills.length} attached
-
- - void handleSkillArchiveUpload(event.target.files)} - /> -
- {skillNotice &&
{skillNotice}
} - {skillError &&
{skillError}
} - {skillInventorySummary.hiddenByRisk > 0 && ( -
- {skillRiskPolicySummary} -
- )} -
-
- - {skillPickerOpen && ( -
-
- {skillOptions.map((skill) => ( - - ))} -
-
- )} -
- -
- {skillEmptyMessage &&
{skillEmptyMessage}
} -
- {instanceSkills.length === 0 ? ( -
- No skills attached. -
- ) : ( -
- {instanceSkills.map((item) => ( -
-
-
- {item.skill?.name || `Skill #${item.skill_id}`} -
-
- {item.status}{item.last_seen_at ? ` - ${formatDateTime(item.last_seen_at, locale)}` : ""} -
-
- -
- ))} -
- )} -
-
- ); - const renderLiteWorkspace = () => ( -
+ const renderLiteWorkspace = () => { + const pinnedWorkspaceHeight = workspaceHeightPx ?? 360; + + return ( +
{renderHeaderSection(shareLinkControl)} {renderActionMessage()} -
-
+
+
{ />
{supportsWorkspace(instance) ? ( -
- fetchSkills(instance.id)} - refreshKey={workspaceRefreshKey} - /> +
+
) : ( -
+
No workspace
)}
- {renderSkillPanel()} +
+ + +
- ); + ); + }; + const runtime = runtimeDetails?.runtime; const agent = runtimeDetails?.agent; const commands = [...(runtimeDetails?.commands ?? [])].sort((left, right) => { @@ -1311,30 +1066,6 @@ const InstanceDetailPage: React.FC = () => { const rightTime = new Date(eventTime(right)).getTime(); return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0); }); - const attachedSkillIds = new Set(instanceSkills.map((item) => item.skill_id)); - const skillOptions = availableSkills.filter((skill) => !attachedSkillIds.has(skill.id)); - const selectedSkillIdSet = new Set(selectedSkillIds); - const skillPickerLabel = selectedSkillIds.length > 0 ? `${selectedSkillIds.length} selected` : "Select skills"; - const skillRiskPolicySummary = locale.startsWith("zh") - ? `\u5b89\u5168\u7b56\u7565\u5df2\u9690\u85cf ${skillInventorySummary.hiddenByRisk} \u4e2a\u4e2d/\u9ad8\u98ce\u9669\u6280\u80fd\uff1b\u5f53\u524d\u53ef\u6dfb\u52a0 ${skillOptions.length} \u4e2a\u3002` - : `Risk policy hid ${skillInventorySummary.hiddenByRisk} medium/high-risk skills; ${skillOptions.length} can still be added.`; - const skillEmptyMessage = - skillOptions.length > 0 - ? null - : skillInventorySummary.total === 0 - ? "No skills loaded from the resource pool." - : skillInventorySummary.hiddenByRisk > 0 - ? "All loaded skills are hidden by risk policy or already attached." - : "All loaded skills are already attached."; const desktopStreamDirty = desktopStreamProfile !== desktopStreamSavedProfile; - const restartActionActive = actionLoading === "restart" || actionLoading === "desktop-stream-restart"; - - const renderActionMessage = () => - actionMessage ? ( -
- - {actionMessage} -
- ) : null; const overviewResourceRows = resourceRows(runtimeDetails, instance).filter((row) => ["CPU", "Memory", "Disk"].includes(row.label), ); @@ -1344,6 +1075,16 @@ const InstanceDetailPage: React.FC = () => { { label: "OpenClaw", value: runtime?.openclaw_status || "-", detail: "Process", percent: null }, ...overviewResourceRows, ]; + const desktopStreamDirty = desktopStreamProfile !== desktopStreamSavedProfile; + const restartActionActive = actionLoading === "restart" || actionLoading === "desktop-stream-restart"; + + const renderActionMessage = () => + actionMessage ? ( +
+ + {actionMessage} +
+ ) : null; const renderProWorkspace = () => (
@@ -1353,7 +1094,7 @@ const InstanceDetailPage: React.FC = () => { data-layout="pro-desktop-workspace" className="grid items-stretch gap-4 xl:grid-cols-[minmax(0,1fr)_minmax(360px,28rem)]" > -
+
{
{supportsWorkspace(instance) ? ( -
- fetchSkills(instance.id)} - refreshKey={workspaceRefreshKey} - /> +
+
) : ( -
+
No workspace
)} @@ -1444,7 +1180,10 @@ const InstanceDetailPage: React.FC = () => { )}
- {renderSkillPanel()} +
@@ -1457,6 +1196,25 @@ const InstanceDetailPage: React.FC = () => {
{runtimeError &&
{runtimeError}
} + {runtimeDetails?.llm_governance && ( +
+ + {runtimeDetails.llm_governance.is_compliant + ? t("instances.governanceGatewayOk") + : runtimeDetails.llm_governance.config_status === "external" + ? t("instances.governanceExternalLLM") + : t("instances.governanceSessionKeyMissing")} + +
+ )}
{runtimeOverviewRows.map((row) => ( @@ -1465,6 +1223,8 @@ const InstanceDetailPage: React.FC = () => {
+ +
@@ -1504,7 +1264,11 @@ const InstanceDetailPage: React.FC = () => { ); return ( - + { const [channelEditorMode, setChannelEditorMode] = useState("form"); const [skillUploadFile, setSkillUploadFile] = useState(null); + const [importPreviewItems, setImportPreviewItems] = useState([]); + const [importDialogOpen, setImportDialogOpen] = useState(false); + const [pendingUploadFile, setPendingUploadFile] = useState(null); const loadAll = async () => { try { @@ -1325,7 +1330,19 @@ const OpenClawConfigCenterPage: React.FC = () => { setSaving(true); setError(null); setNotice(null); - await skillService.importSkills(skillUploadFile); + const preview = await skillHubService.previewImportSkills(skillUploadFile); + const conflicts = preview.filter((item) => item.conflict_type === "content_changed"); + if (conflicts.length > 0) { + setPendingUploadFile(skillUploadFile); + setImportPreviewItems(preview); + setImportDialogOpen(true); + return; + } + const decisions: SkillImportDecision[] = preview.map((item) => ({ + directory_name: item.directory_name, + action: item.conflict_type === "unchanged" ? "skip" : "new_version", + })); + await skillHubService.importSkills(skillUploadFile, decisions); setSkillUploadFile(null); await loadAll(); setNotice(t("openClawResourcesPage.notices.skillArchiveImported")); @@ -1339,6 +1356,36 @@ const OpenClawConfigCenterPage: React.FC = () => { } }; + const handleImportConflictConfirm = async (decisions: SkillImportDecision[]) => { + if (!pendingUploadFile) { + return; + } + try { + setSaving(true); + setError(null); + await skillHubService.importSkills(pendingUploadFile, decisions); + setSkillUploadFile(null); + setPendingUploadFile(null); + setImportPreviewItems([]); + setImportDialogOpen(false); + await loadAll(); + setNotice(t("openClawResourcesPage.notices.skillArchiveImported")); + } catch (err: any) { + setError( + err.response?.data?.error || + t("openClawResourcesPage.errors.importSkillArchive"), + ); + } finally { + setSaving(false); + } + }; + + const handleImportConflictCancel = () => { + setImportDialogOpen(false); + setPendingUploadFile(null); + setImportPreviewItems([]); + }; + const removeSkillAsset = async (skillId: number) => { try { setSaving(true); @@ -1628,6 +1675,12 @@ const OpenClawConfigCenterPage: React.FC = () => {
{t("openClawResourcesPage.skillUploadHint")}
+
+ {t("skillHubPage.configCenterHint")}{" "} + + {t("skillHubPage.goToHub")} + +
{loading ? (
@@ -2644,6 +2697,13 @@ const OpenClawConfigCenterPage: React.FC = () => {
+ void handleImportConflictConfirm(decisions)} + onCancel={handleImportConflictCancel} + /> ); }; diff --git a/frontend/src/pages/skill-hub/SkillHubPage.tsx b/frontend/src/pages/skill-hub/SkillHubPage.tsx new file mode 100644 index 0000000..0a4c320 --- /dev/null +++ b/frontend/src/pages/skill-hub/SkillHubPage.tsx @@ -0,0 +1,675 @@ +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import SkillImportConflictDialog from '../../components/SkillImportConflictDialog'; +import UserLayout from '../../components/UserLayout'; +import { useAuth } from '../../contexts/AuthContext'; +import { useI18n } from '../../contexts/I18nContext'; +import { instanceService } from '../../services/instanceService'; +import { skillHubService } from '../../services/skillHubService'; +import type { Instance } from '../../types/instance'; +import type { + Skill, + SkillHubTag, + SkillImportDecision, + SkillImportPreviewItem, + SkillImportResultItem, +} from '../../types/skill'; + +type HubTab = 'catalog' | 'mine' | 'admin'; + +const SkillHubPage: React.FC = () => { + const { user } = useAuth(); + const { t } = useI18n(); + const [tab, setTab] = useState('catalog'); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); + const [tags, setTags] = useState([]); + const [catalogSkills, setCatalogSkills] = useState([]); + const [mySkills, setMySkills] = useState([]); + const [adminSkills, setAdminSkills] = useState([]); + const [instances, setInstances] = useState([]); + const [search, setSearch] = useState(''); + const [selectedTag, setSelectedTag] = useState(''); + const [uploadFiles, setUploadFiles] = useState([]); + const [publishSkillId, setPublishSkillId] = useState(null); + const [editTagsSkillId, setEditTagsSkillId] = useState(null); + const [selectedTagIds, setSelectedTagIds] = useState([]); + const [installSkillId, setInstallSkillId] = useState(null); + const [selectedInstanceId, setSelectedInstanceId] = useState(''); + const [actionLoading, setActionLoading] = useState(''); + const [importPreviewItems, setImportPreviewItems] = useState([]); + const [importDialogOpen, setImportDialogOpen] = useState(false); + const [pendingUploadFile, setPendingUploadFile] = useState(null); + + const tagLabel = useCallback( + (tag: SkillHubTag) => t(`skillHubPage.tags.${tag.tag_key}`) || tag.name, + [t], + ); + + const loadBase = useCallback(async () => { + setLoading(true); + setError(null); + try { + const [tagItems, instanceResponse] = await Promise.all([ + skillHubService.listTags(), + instanceService.getInstances(1, 100), + ]); + setTags(tagItems); + setInstances(instanceResponse.instances || []); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : t('skillHubPage.errors.load'); + setError(message); + } finally { + setLoading(false); + } + }, [t]); + + const loadCatalog = useCallback(async () => { + try { + const result = await skillHubService.listCatalog({ + q: search || undefined, + tag_keys: selectedTag ? [selectedTag] : undefined, + page: 1, + page_size: 100, + }); + setCatalogSkills(result.items || []); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : t('skillHubPage.errors.load'); + setError(message); + } + }, [search, selectedTag, t]); + + const loadMine = useCallback(async () => { + try { + const items = await skillHubService.listMine(); + setMySkills(items); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : t('skillHubPage.errors.load'); + setError(message); + } + }, [t]); + + const loadAdmin = useCallback(async () => { + if (user?.role !== 'admin') { + return; + } + try { + const items = await skillHubService.listAdminSkills(); + setAdminSkills(items); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : t('skillHubPage.errors.load'); + setError(message); + } + }, [t, user?.role]); + + const refreshAll = useCallback(async () => { + await loadBase(); + await Promise.all([loadCatalog(), loadMine(), loadAdmin()]); + }, [loadAdmin, loadBase, loadCatalog, loadMine]); + + useEffect(() => { + void loadBase(); + void loadMine(); + }, [loadBase, loadMine]); + + useEffect(() => { + if (tab === 'catalog') { + void loadCatalog(); + } else if (tab === 'mine') { + void loadMine(); + } else if (tab === 'admin') { + void loadAdmin(); + } + }, [tab, loadAdmin, loadCatalog, loadMine]); + + useEffect(() => { + const onVisibilityChange = () => { + if (!document.hidden) { + void refreshAll(); + } + }; + document.addEventListener('visibilitychange', onVisibilityChange); + return () => { + document.removeEventListener('visibilitychange', onVisibilityChange); + }; + }, [refreshAll]); + + const visibleTags = useMemo( + () => tags.filter((tag) => !tag.admin_only || user?.role === 'admin'), + [tags, user?.role], + ); + + const currentSkills = tab === 'catalog' ? catalogSkills : tab === 'mine' ? mySkills : adminSkills; + + const toggleTag = (tagId: number) => { + setSelectedTagIds((current) => + current.includes(tagId) ? current.filter((id) => id !== tagId) : [...current, tagId], + ); + }; + + const buildDefaultDecisions = (preview: SkillImportPreviewItem[]): SkillImportDecision[] => + preview.map((item) => { + if (item.conflict_type === 'unchanged') { + return { directory_name: item.directory_name, action: 'skip' }; + } + return { directory_name: item.directory_name, action: 'new_version' }; + }); + + const formatImportNotice = (results: SkillImportResultItem[]): string => { + const messages = results.map((item) => { + const name = item.skill.name || item.skill.skill_key; + switch (item.action) { + case 'created': + return t('skillHubPage.notices.created', { name }); + case 'versioned': + return t('skillHubPage.notices.versioned', { + name, + version: item.skill.current_version_no ?? (item.previous_version_no ?? 0) + 1, + }); + case 'unchanged': + return t('skillHubPage.notices.unchanged', { name }); + case 'saved_as_new': + return t('skillHubPage.notices.savedAsNew', { key: item.skill.skill_key }); + default: + return t('skillHubPage.notices.uploaded'); + } + }); + return messages.join(' · '); + }; + + const finalizeImport = async (file: File, decisions: SkillImportDecision[]) => { + return skillHubService.importSkills(file, decisions); + }; + + const importSingleArchive = async (file: File): Promise => { + const preview = await skillHubService.previewImportSkills(file); + const conflicts = preview.filter((item) => item.conflict_type === 'content_changed'); + if (conflicts.length === 0) { + return finalizeImport(file, buildDefaultDecisions(preview)); + } + return new Promise((resolve, reject) => { + setPendingUploadFile(file); + setImportPreviewItems(preview); + setImportDialogOpen(true); + pendingImportResolverRef.current = { resolve, reject }; + }); + }; + + const pendingImportResolverRef = useRef<{ + resolve: (value: SkillImportResultItem[]) => void; + reject: (reason?: unknown) => void; + } | null>(null); + + const handleUpload = async () => { + if (uploadFiles.length === 0) { + return; + } + try { + setActionLoading('upload'); + setError(null); + const allResults: SkillImportResultItem[] = []; + const errors: string[] = []; + for (const file of uploadFiles) { + try { + const results = await importSingleArchive(file); + allResults.push(...results); + } catch (err: unknown) { + errors.push(`${file.name}: ${(err as { response?: { data?: { error?: string } } })?.response?.data?.error || t('skillHubPage.errors.upload')}`); + } + } + if (allResults.length > 0) { + setUploadFiles([]); + setNotice(formatImportNotice(allResults)); + setTab('mine'); + await loadMine(); + } + if (errors.length > 0) { + setError(errors.join(' · ')); + } + } catch (err: unknown) { + setError((err as { response?: { data?: { error?: string } } })?.response?.data?.error || t('skillHubPage.errors.upload')); + } finally { + setActionLoading(''); + } + }; + + const handleImportConflictConfirm = async (decisions: SkillImportDecision[]) => { + if (!pendingUploadFile) { + return; + } + try { + setActionLoading('upload'); + setError(null); + const results = await finalizeImport(pendingUploadFile, decisions); + pendingImportResolverRef.current?.resolve(results); + pendingImportResolverRef.current = null; + setPendingUploadFile(null); + setImportPreviewItems([]); + setImportDialogOpen(false); + } catch (err: unknown) { + pendingImportResolverRef.current?.reject(err); + pendingImportResolverRef.current = null; + setError((err as { response?: { data?: { error?: string } } })?.response?.data?.error || t('skillHubPage.errors.upload')); + } finally { + setActionLoading(''); + } + }; + + const handleImportConflictCancel = () => { + pendingImportResolverRef.current?.reject(new Error('import_cancelled')); + pendingImportResolverRef.current = null; + setImportDialogOpen(false); + setPendingUploadFile(null); + setImportPreviewItems([]); + }; + + const handlePublish = async () => { + if (!publishSkillId || selectedTagIds.length === 0) { + setError(t('skillHubPage.errors.tagsRequired')); + return; + } + try { + setActionLoading(`publish-${publishSkillId}`); + setError(null); + await skillHubService.publishSkill(publishSkillId, selectedTagIds); + setPublishSkillId(null); + setSelectedTagIds([]); + setNotice(t('skillHubPage.notices.published')); + await refreshAll(); + } catch (err: unknown) { + setError((err as { response?: { data?: { error?: string } } })?.response?.data?.error || t('skillHubPage.errors.publish')); + } finally { + setActionLoading(''); + } + }; + + const handleUpdateTags = async () => { + if (!editTagsSkillId || selectedTagIds.length === 0) { + setError(t('skillHubPage.errors.tagsRequired')); + return; + } + try { + setActionLoading(`edit-tags-${editTagsSkillId}`); + setError(null); + await skillHubService.updateTags(editTagsSkillId, selectedTagIds); + setEditTagsSkillId(null); + setSelectedTagIds([]); + setNotice(t('skillHubPage.notices.tagsUpdated')); + await refreshAll(); + } catch (err: unknown) { + setError((err as { response?: { data?: { error?: string } } })?.response?.data?.error || t('skillHubPage.errors.updateTags')); + } finally { + setActionLoading(''); + } + }; + + const handleUnpublish = async (skillId: number) => { + try { + setActionLoading(`unpublish-${skillId}`); + await skillHubService.unpublishSkill(skillId); + setNotice(t('skillHubPage.notices.unpublished')); + await refreshAll(); + } catch (err: unknown) { + setError((err as { response?: { data?: { error?: string } } })?.response?.data?.error || t('skillHubPage.errors.publish')); + } finally { + setActionLoading(''); + } + }; + + const handleDownload = async (skill: Skill) => { + try { + setActionLoading(`download-${skill.id}`); + setError(null); + const blob = await skillHubService.downloadSkill(skill.id); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = `${skill.skill_key}.zip`; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); + } catch (err: unknown) { + setError((err as { response?: { data?: { error?: string } } })?.response?.data?.error || t('skillHubPage.errors.download')); + } finally { + setActionLoading(''); + } + }; + + const handleDelete = async (skillId: number) => { + try { + setActionLoading(`delete-${skillId}`); + await skillHubService.deleteSkill(skillId); + setNotice(t('skillHubPage.notices.deleted')); + await refreshAll(); + } catch (err: unknown) { + setError((err as { response?: { data?: { error?: string } } })?.response?.data?.error || t('skillHubPage.errors.delete')); + } finally { + setActionLoading(''); + } + }; + + const handleInstall = async () => { + if (!installSkillId || selectedInstanceId === '') { + return; + } + try { + setActionLoading(`install-${installSkillId}`); + await skillHubService.installSkill(installSkillId, Number(selectedInstanceId)); + setInstallSkillId(null); + setSelectedInstanceId(''); + setNotice(t('skillHubPage.notices.installed')); + } catch (err: unknown) { + setError((err as { response?: { data?: { error?: string } } })?.response?.data?.error || t('skillHubPage.errors.install')); + } finally { + setActionLoading(''); + } + }; + + const blockReasonLabel = (reason?: string) => { + if (!reason) { + return null; + } + const key = `skillHubPage.blockReasons.${reason}`; + const label = t(key); + return label === key ? reason : label; + }; + + const renderSkillCard = (skill: Skill, options?: { showOwner?: boolean; adminView?: boolean }) => { + const isOwner = skill.user_id === user?.id; + const canPublishManage = isOwner; + const canDelete = isOwner || (options?.adminView === true && user?.role === 'admin'); + return ( +
+
+
+
+

{skill.name}

+ {skill.current_version_no ? ( + + {t('skillHubPage.versionLabel', { version: skill.current_version_no })} + + ) : null} + + {skill.visibility === 'public' ? t('skillHubPage.visibility.public') : t('skillHubPage.visibility.private')} + + + {skill.risk_level} + + {skill.scan_status ? ( + + {skill.scan_status} + + ) : null} +
+

{skill.skill_key}

+ {options?.showOwner && skill.owner_username ? ( +

{t('skillHubPage.owner')}: {skill.owner_username}

+ ) : null} + {skill.published_at ? ( +

+ {t('skillHubPage.publishedAt')}: {new Date(skill.published_at).toLocaleString()} +

+ ) : null} +
+ {(skill.tags || []).map((tag) => ( + + {tagLabel(tag)} + + ))} +
+

+ {skill.publishable ? t('skillHubPage.publishable') : t('skillHubPage.notPublishable')} + {!skill.publishable && skill.publish_blocked_reason ? ( + <> · {blockReasonLabel(skill.publish_blocked_reason)} + ) : null} + {' · '} + {t('skillHubPage.instances')}: {skill.instance_count} +

+ {skill.package_collect_error ? ( +

{skill.package_collect_error}

+ ) : null} + {!skill.publishable && skill.source_type === 'discovered' ? ( +

{t('skillHubPage.discoveredPublishHint')}

+ ) : null} +
+
+ {tab === 'catalog' ? ( + <> + + + + ) : null} + {(tab === 'mine' || tab === 'admin') && skill.visibility === 'public' ? ( + + ) : null} + {canPublishManage && skill.visibility !== 'public' ? ( + + ) : null} + {canPublishManage && skill.visibility === 'public' ? ( + <> + + + + ) : null} + {canDelete ? ( + + ) : null} +
+
+
+ ); + }; + + return ( + +
+
+

Skill Hub

+

{t('skillHubPage.title')}

+

{t('skillHubPage.subtitle')}

+
+ + {error ?
{error}
: null} + {notice ?
{notice}
: null} + +
+
+ {(['catalog', 'mine', ...(user?.role === 'admin' ? ['admin'] as const : [])] as HubTab[]).map((item) => ( + + ))} +
+ + {tab === 'catalog' ? ( +
+ setSearch(event.target.value)} + /> + + +
+ ) : null} + + {tab === 'mine' ? ( +
+ + +
+ ) : null} + +
+ {loading ?
{t('common.loading')}
: null} + {!loading && currentSkills.length === 0 ? ( +
+ {t('skillHubPage.noSkills')} +
+ ) : null} + {!loading ? currentSkills.map((skill) => renderSkillCard(skill, { showOwner: tab !== 'mine', adminView: tab === 'admin' })) : null} +
+
+
+ + {publishSkillId !== null ? ( +
+
+

{t('skillHubPage.publish')}

+

{t('skillHubPage.selectTags')}

+
+ {visibleTags.map((tag) => ( + + ))} +
+
+ + +
+
+
+ ) : null} + + {editTagsSkillId !== null ? ( +
+
+

{t('skillHubPage.editTags')}

+

{t('skillHubPage.selectTags')}

+
+ {visibleTags.map((tag) => ( + + ))} +
+
+ + +
+
+
+ ) : null} + + {installSkillId !== null ? ( +
+
+

{t('skillHubPage.install')}

+ +
+ + +
+
+
+ ) : null} + + void handleImportConflictConfirm(decisions)} + onCancel={handleImportConflictCancel} + /> + + ); +}; + +export default SkillHubPage; diff --git a/frontend/src/router/index.tsx b/frontend/src/router/index.tsx index 7add2b3..623b088 100644 --- a/frontend/src/router/index.tsx +++ b/frontend/src/router/index.tsx @@ -17,6 +17,7 @@ import InstanceManagementPage from '../pages/admin/InstanceManagementPage'; import AIGatewayPage from '../pages/admin/AIGatewayPage'; import AIAuditPage from '../pages/admin/AIAuditPage'; import CostsPage from '../pages/admin/CostsPage'; +import SessionUsageOverviewPage from '../pages/admin/SessionUsageOverviewPage'; import AdminSecurityDashboardPage from '../pages/admin/security/AdminSecurityDashboardPage'; import AdminSecurityReportsPage from '../pages/admin/security/AdminSecurityReportsPage'; import AdminSecurityScannerConfigPage from '../pages/admin/security/AdminSecurityScannerConfigPage'; @@ -55,6 +56,7 @@ import InstancePortalPage from '../pages/instances/InstancePortalPage'; import TeamListPage from '../pages/teams/TeamListPage'; import CreateTeamPage from '../pages/teams/CreateTeamPage'; import TeamDetailPage from '../pages/teams/TeamDetailPage'; +import SkillHubPage from '../pages/skill-hub/SkillHubPage'; // Protected Route Component const ProtectedRoute: React.FC<{ children: React.ReactNode }> = ({ children }) => { @@ -239,6 +241,14 @@ function AppRoutes() { } /> + + + + } + /> } /> + + + + } + /> => { const response = await api.get('/admin/skills'); @@ -480,4 +512,29 @@ export const adminService = { limit: data.limit ?? params?.limit ?? 20, }; }, + + getSessionUsageOverview: async (params?: { + page?: number; + limit?: number; + search?: string; + since?: string; + until?: string; + }): Promise => { + const response = await api.get("/admin/session-usage/overview", { params }); + const data = response.data.data ?? {}; + return { + summary: data.summary ?? { + total_prompt_tokens: 0, + total_completion_tokens: 0, + total_tokens: 0, + total_estimated_cost: 0, + currency: "USD", + session_count: 0, + }, + items: data.items ?? [], + total: data.total ?? 0, + page: data.page ?? params?.page ?? 1, + limit: data.limit ?? params?.limit ?? 20, + }; + }, }; diff --git a/frontend/src/services/instanceService.ts b/frontend/src/services/instanceService.ts index d5f4bfc..0983351 100644 --- a/frontend/src/services/instanceService.ts +++ b/frontend/src/services/instanceService.ts @@ -6,6 +6,7 @@ import type { UpdateInstanceRequest, InstanceStatus, InstanceRuntimeDetails, + InstanceRuntimeCommand, InstanceConfigRevision, ExternalAccessStatusResult, EnableShareLinkResult, @@ -14,6 +15,8 @@ import type { BatchCreateLiteInstancesRequest, BatchCreateLiteInstancesResponse, BatchDeleteLiteInstancesResponse, + InstanceSessionUsageDetail, + InstanceSessionUsageResult, } from "../types/instance"; import type { InstanceSkill } from "../types/skill"; @@ -231,4 +234,28 @@ export const instanceService = { const response = await api.get(`/instances/${id}/skills`); return response.data.data; }, + + syncInstanceSkills: async (id: number): Promise => { + const response = await api.post(`/instances/${id}/skills/sync`); + return response.data.data; + }, + + getInstanceSessionUsage: async ( + id: number, + params?: { page?: number; limit?: number; search?: string; since?: string; until?: string }, + ): Promise => { + const response = await api.get(`/instances/${id}/session-usage`, { params }); + return response.data.data; + }, + + getInstanceSessionUsageDetail: async ( + id: number, + sessionId: string, + params?: { since?: string; until?: string }, + ): Promise => { + const response = await api.get(`/instances/${id}/session-usage/detail`, { + params: { session_id: sessionId, ...params }, + }); + return response.data.data; + }, }; diff --git a/frontend/src/services/skillHubService.ts b/frontend/src/services/skillHubService.ts new file mode 100644 index 0000000..3860e5d --- /dev/null +++ b/frontend/src/services/skillHubService.ts @@ -0,0 +1,101 @@ +import api from './api'; +import type { + InstanceSkill, + Skill, + SkillHubCatalogResponse, + SkillHubTag, + SkillImportDecision, + SkillImportPreviewItem, + SkillImportResultItem, +} from '../types/skill'; + +export const skillHubService = { + listCatalog: async (params?: { tag_keys?: string[]; q?: string; page?: number; page_size?: number }): Promise => { + const response = await api.get('/skill-hub/catalog', { params }); + return response.data.data; + }, + + listTags: async (): Promise => { + const response = await api.get('/skill-hub/tags'); + return response.data.data; + }, + + listMine: async (): Promise => { + const response = await api.get('/skill-hub/mine'); + return response.data.data; + }, + + listAttachable: async (): Promise => { + const response = await api.get('/skill-hub/attachable'); + return response.data.data; + }, + + listAdminSkills: async (): Promise => { + const response = await api.get('/admin/skill-hub/skills'); + return response.data.data; + }, + + previewImportSkills: async (file: File): Promise => { + const formData = new FormData(); + formData.append('file', file); + const response = await api.post('/skill-hub/skills/import/preview', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }); + return response.data.data; + }, + + importSkills: async (file: File, decisions?: SkillImportDecision[]): Promise => { + const formData = new FormData(); + formData.append('file', file); + if (decisions && decisions.length > 0) { + formData.append('decisions', JSON.stringify(decisions)); + } + const response = await api.post('/skill-hub/skills/import', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }); + return response.data.data; + }, + + publishSkill: async (skillId: number, tagIds: number[]): Promise => { + const response = await api.post(`/skill-hub/skills/${skillId}/publish`, { tag_ids: tagIds }); + return response.data.data; + }, + + unpublishSkill: async (skillId: number): Promise => { + const response = await api.post(`/skill-hub/skills/${skillId}/unpublish`); + return response.data.data; + }, + + updateTags: async (skillId: number, tagIds: number[]): Promise => { + const response = await api.put(`/skill-hub/skills/${skillId}/tags`, { tag_ids: tagIds }); + return response.data.data; + }, + + deleteSkill: async (skillId: number): Promise => { + await api.delete(`/skill-hub/skills/${skillId}`); + }, + + downloadSkill: async (skillId: number): Promise => { + const response = await api.get(`/skill-hub/skills/${skillId}/download`, { responseType: 'blob' }); + return response.data; + }, + + installSkill: async (skillId: number, instanceId: number): Promise => { + const response = await api.post(`/skill-hub/skills/${skillId}/install`, { instance_id: instanceId }); + return response.data.data; + }, + + publishFromInstance: async (instanceId: number, skillId: number, tagIds: number[]): Promise => { + const response = await api.post(`/instances/${instanceId}/skills/${skillId}/publish-to-hub`, { tag_ids: tagIds }); + return response.data.data; + }, + + importInstanceSkill: async (instanceId: number, skillId: number): Promise => { + const response = await api.post(`/instances/${instanceId}/skills/${skillId}/import-to-library`); + return response.data.data; + }, + + retryPackageCollect: async (instanceId: number, skillId: number): Promise => { + await api.post(`/instances/${instanceId}/skills/${skillId}/retry-package-collect`); + }, +}; diff --git a/frontend/src/types/instance.ts b/frontend/src/types/instance.ts index adfaae9..2f45b71 100644 --- a/frontend/src/types/instance.ts +++ b/frontend/src/types/instance.ts @@ -150,6 +150,7 @@ export interface InstanceRuntimeDetails { agent?: AgentInfo; commands: InstanceRuntimeCommand[]; skills?: InstanceSkill[]; + llm_governance?: InstanceLLMGovernanceStatus; } export interface InstanceConfigRevision { @@ -317,6 +318,83 @@ export const INSTANCE_TYPES: InstanceType[] = [ }, ]; +export interface InstanceSessionUsageSummary { + total_prompt_tokens: number; + total_completion_tokens: number; + total_tokens: number; + total_estimated_cost: number; + currency: string; + session_count: number; +} + +export interface InstanceSessionUsageCompliance { + fallback_session_count: number; + has_fallback_sessions: boolean; + recent_fallback_audit_count: number; +} + +export interface InstanceSessionUsageItem { + session_id: string; + session_key: string; + title?: string; + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + estimated_cost: number; + currency: string; + invocation_count: number; + first_seen_at: string; + last_seen_at: string; +} + +export interface InstanceSessionUsageResult { + summary: InstanceSessionUsageSummary; + compliance: InstanceSessionUsageCompliance; + items: InstanceSessionUsageItem[]; + total: number; + page: number; + limit: number; +} + +export interface InstanceSessionTrace { + trace_id: string; + requested_model: string; + status: string; + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + created_at: string; +} + +export interface InstanceSessionUsageDetail { + session_id: string; + session_key: string; + title?: string; + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + estimated_cost: number; + currency: string; + invocation_count: number; + first_seen_at: string; + last_seen_at: string; + model_breakdown: Array<{ + label: string; + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + estimated_cost: number; + }>; + recent_traces: InstanceSessionTrace[]; +} + +export interface InstanceLLMGovernanceStatus { + config_status: string; + session_fallback_rate: number; + recent_egress_block_count: number; + is_compliant: boolean; +} + export const PRESET_CONFIGS = { small: { name: "Small", diff --git a/frontend/src/types/skill.ts b/frontend/src/types/skill.ts index c97c6d4..191ec4c 100644 --- a/frontend/src/types/skill.ts +++ b/frontend/src/types/skill.ts @@ -7,16 +7,74 @@ export interface Skill { status: string; source_type: string; risk_level: string; + scan_status?: string; last_scanned_at?: string; current_version_id?: number; current_version_no?: number; content_hash?: string; archive_hash?: string; instance_count: number; + visibility?: string; + published_at?: string; + published_by?: number; + tags?: SkillHubTag[]; + publishable?: boolean; + publish_blocked_reason?: string; + package_collect_error?: string; + package_materialize_status?: string; + package_materialize_error?: string; + owner_username?: string; created_at: string; updated_at: string; } +export interface SkillHubTag { + id: number; + tag_key: string; + name: string; + description?: string; + sort_order: number; + admin_only: boolean; +} + +export interface SkillHubCatalogResponse { + items: Skill[]; + total: number; + page: number; + page_size: number; + total_pages: number; +} + +export type SkillImportConflictType = 'none' | 'unchanged' | 'content_changed'; + +export interface SkillImportPreviewItem { + directory_name: string; + skill_key: string; + content_hash: string; + conflict_type: SkillImportConflictType; + existing_skill_id?: number; + existing_name?: string; + current_version_no?: number; + suggested_skill_key?: string; +} + +export type SkillImportDecisionAction = 'new_version' | 'save_as_new' | 'skip'; + +export interface SkillImportDecision { + directory_name: string; + action: SkillImportDecisionAction; + skill_key?: string; +} + +export type SkillImportResultAction = 'created' | 'versioned' | 'unchanged' | 'saved_as_new'; + +export interface SkillImportResultItem { + skill: Skill; + action: SkillImportResultAction; + previous_version_no?: number; + directory_name: string; +} + export interface SkillVersion { id: number; skill_id: number; @@ -49,6 +107,7 @@ export interface InstanceSkill { skill_version_id?: number; source_type: string; install_path?: string; + workspace_dir?: string; observed_hash?: string; status: string; last_seen_at?: string; diff --git a/frontend/src/utils/sessionUsageExport.ts b/frontend/src/utils/sessionUsageExport.ts new file mode 100644 index 0000000..5009398 --- /dev/null +++ b/frontend/src/utils/sessionUsageExport.ts @@ -0,0 +1,66 @@ +import type { InstanceSessionUsageItem } from "../types/instance"; + +export type SessionUsageTimeRange = "all" | "24h" | "7d" | "30d"; + +export function resolveSessionUsageSince(range: SessionUsageTimeRange): string | undefined { + if (range === "all") { + return undefined; + } + const now = Date.now(); + const offsets: Record, number> = { + "24h": 24 * 60 * 60 * 1000, + "7d": 7 * 24 * 60 * 60 * 1000, + "30d": 30 * 24 * 60 * 60 * 1000, + }; + return new Date(now - offsets[range]).toISOString(); +} + +function escapeCsvValue(value: string | number | undefined | null): string { + const normalized = value == null ? "" : String(value); + if (/[",\n]/.test(normalized)) { + return `"${normalized.replace(/"/g, '""')}"`; + } + return normalized; +} + +export function buildSessionUsageCsv(items: InstanceSessionUsageItem[], currency: string): string { + const header = [ + "session_id", + "session_key", + "title", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "estimated_cost", + "currency", + "invocation_count", + "first_seen_at", + "last_seen_at", + ]; + const rows = items.map((item) => [ + item.session_id, + item.session_key, + item.title ?? "", + item.prompt_tokens, + item.completion_tokens, + item.total_tokens, + item.estimated_cost, + item.currency || currency, + item.invocation_count, + item.first_seen_at, + item.last_seen_at, + ]); + return [header, ...rows] + .map((row) => row.map((cell) => escapeCsvValue(cell)).join(",")) + .join("\n"); +} + +export function downloadSessionUsageCsv(content: string, filename: string) { + const blob = new Blob(["\uFEFF", content], { type: "text/csv;charset=utf-8;" }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = filename; + link.click(); + URL.revokeObjectURL(url); +}