feat:Improve Lite gateway scheduling, proxy stability, and Hermes recovery (#160)
* feat(runtime): improve Lite gateway scheduling and proxy stability - Expand the runtime gateway port range to match 100 instances per pod - Scale runtime deployments based on pending backlog - Serialize gateway creation per pod and handle starting/creating states - Clean up missing gateway bindings from runtime agent reports - Improve Lite gateway proxy token stripping and auth header injection - Update deployment manifests, docs, and related tests * fix(runtime): harden Hermes Lite gateway recovery - Add hashed gateway token aliases for short-term token compatibility - Fix /resume conversations failing when old sessions use mismatched gateway tokens - Recover ports only from stale error bindings without affecting active gateways * feat(runtime): enable bidirectional desktop clipboard and direct proxy - Enable WebTop/Selkies/Kasm clipboard by default with per-instance overrides - Document clipboard verification, configuration pitfalls, and IME troubleshooting - Enable direct desktop proxy in the K8s and K3s manifests --------- Co-authored-by: litiantian03 <litiantian03@ieisystem.com>
This commit is contained in:
@@ -83,3 +83,4 @@ TEAM_PROFILES_LOCAL_TESTING.md
|
||||
/clawmanager-apply.sh
|
||||
/clawmanager-team-profiles-test.yaml
|
||||
/clawmanager-tenant.yaml
|
||||
.codex
|
||||
|
||||
@@ -286,7 +286,7 @@ func Load() (*Config, error) {
|
||||
HermesImage: getEnv("HERMES_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"),
|
||||
MaxGatewaysPerPod: getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", 100),
|
||||
GatewayPortStart: getEnvInt("RUNTIME_GATEWAY_PORT_START", 20000),
|
||||
GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", 20099),
|
||||
GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", 20299),
|
||||
},
|
||||
ObjectStorage: ObjectStorageConfig{
|
||||
Endpoint: getEnv("OBJECT_STORAGE_ENDPOINT", ""),
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE IF NOT EXISTS instance_gateway_token_aliases (
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
instance_id INT NOT NULL,
|
||||
token_hash CHAR(64) NOT NULL,
|
||||
expires_at TIMESTAMP NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
last_used_at TIMESTAMP NULL,
|
||||
CONSTRAINT fk_instance_gateway_token_aliases_instance
|
||||
FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE,
|
||||
UNIQUE KEY uk_instance_gateway_token_aliases_hash (token_hash),
|
||||
INDEX idx_instance_gateway_token_aliases_instance (instance_id),
|
||||
INDEX idx_instance_gateway_token_aliases_expires (expires_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
@@ -160,3 +160,26 @@ func TestMigration037AddsTeamWorkflowLedger(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigration038AddsGatewayTokenAliases(t *testing.T) {
|
||||
raw, err := embeddedMigrations.ReadFile("migrations/038_add_instance_gateway_token_aliases.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read migration 038: %v", err)
|
||||
}
|
||||
sql := string(raw)
|
||||
for _, required := range []string{
|
||||
"CREATE TABLE IF NOT EXISTS instance_gateway_token_aliases",
|
||||
"token_hash CHAR(64)",
|
||||
"expires_at TIMESTAMP NOT NULL",
|
||||
"last_used_at TIMESTAMP NULL",
|
||||
"uk_instance_gateway_token_aliases_hash",
|
||||
"ON DELETE CASCADE",
|
||||
} {
|
||||
if !strings.Contains(sql, required) {
|
||||
t.Fatalf("migration 038 must contain %s", required)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "access_token") {
|
||||
t.Fatalf("migration 038 must not store raw access tokens")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,6 @@ const openclawMinArchiveBytes = 100
|
||||
const (
|
||||
defaultWorkspaceArchiveMaxMiB = int64(500)
|
||||
workspaceArchiveMaxMiBEnv = "CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB"
|
||||
maxLiteBatchCreateCount = 20
|
||||
maxLiteBatchDeleteCount = 100
|
||||
)
|
||||
|
||||
@@ -226,7 +225,7 @@ type BatchCreateLiteInstanceTemplate struct {
|
||||
// BatchCreateLiteInstancesRequest represents a lite batch create request
|
||||
type BatchCreateLiteInstancesRequest struct {
|
||||
NamePrefix string `json:"name_prefix" binding:"required,min=1,max=40"`
|
||||
Count int `json:"count" binding:"required,min=1,max=20"`
|
||||
Count int `json:"count" binding:"required,min=1"`
|
||||
StartIndex int `json:"start_index" binding:"omitempty,min=0,max=9999"`
|
||||
Template *BatchCreateLiteInstanceTemplate `json:"template,omitempty"`
|
||||
}
|
||||
@@ -479,9 +478,6 @@ func buildLiteBatchCreateRequests(req BatchCreateLiteInstancesRequest) ([]servic
|
||||
if count <= 0 {
|
||||
return nil, nil, fmt.Errorf("count is required")
|
||||
}
|
||||
if count > maxLiteBatchCreateCount {
|
||||
return nil, nil, fmt.Errorf("count must be at most %d", maxLiteBatchCreateCount)
|
||||
}
|
||||
|
||||
prefix := strings.TrimSpace(req.NamePrefix)
|
||||
if prefix == "" {
|
||||
|
||||
@@ -272,7 +272,9 @@ func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
reported := make(map[int]int, len(req.Gateways))
|
||||
for _, gateway := range req.Gateways {
|
||||
reported[gateway.InstanceID] = gateway.Generation
|
||||
binding, err := h.bindingRepo.GetByInstanceID(c.Request.Context(), gateway.InstanceID)
|
||||
if err != nil {
|
||||
utils.HandleError(c, err)
|
||||
@@ -281,20 +283,37 @@ func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) {
|
||||
if binding == nil || binding.RuntimePodID != podID || binding.Generation != gateway.Generation {
|
||||
continue
|
||||
}
|
||||
state := strings.ToLower(strings.TrimSpace(gateway.State))
|
||||
switch state {
|
||||
case "running", "ready", "healthy":
|
||||
lifecycle := services.NormalizeRuntimeGatewayLifecycle(gateway.State, gateway.ErrorMessage)
|
||||
if lifecycle.Running {
|
||||
if err := h.bindingRepo.UpdateRunning(c.Request.Context(), gateway.InstanceID, gateway.Generation, strings.TrimSpace(gateway.GatewayID), gateway.GatewayPort, gateway.GatewayPID); err != nil {
|
||||
utils.HandleError(c, err)
|
||||
return
|
||||
}
|
||||
default:
|
||||
if err := h.bindingRepo.UpdateState(c.Request.Context(), gateway.InstanceID, gateway.Generation, state, gateway.ErrorMessage); err != nil {
|
||||
} else {
|
||||
if err := h.bindingRepo.UpdateState(c.Request.Context(), gateway.InstanceID, gateway.Generation, lifecycle.BindingState, lifecycle.Message); err != nil {
|
||||
utils.HandleError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := h.syncInstanceRuntimeState(c.Request.Context(), gateway, state); err != nil {
|
||||
if err := h.syncInstanceRuntimeState(c.Request.Context(), gateway, lifecycle); err != nil {
|
||||
utils.HandleError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
bindings, err := h.bindingRepo.ListByRuntimePodID(c.Request.Context(), podID)
|
||||
if err != nil {
|
||||
utils.HandleError(c, err)
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, binding := range bindings {
|
||||
if _, reportedByInstance := reported[binding.InstanceID]; reportedByInstance {
|
||||
continue
|
||||
}
|
||||
if !h.missingGatewayCleanupEligible(binding, now) {
|
||||
continue
|
||||
}
|
||||
if _, err := h.bindingRepo.DeleteRunningByInstanceIDGenerationAndReleaseSlot(c.Request.Context(), binding.InstanceID, podID, binding.Generation); err != nil {
|
||||
utils.HandleError(c, err)
|
||||
return
|
||||
}
|
||||
@@ -306,35 +325,21 @@ func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) {
|
||||
utils.Success(c, http.StatusOK, "Runtime gateway report accepted", nil)
|
||||
}
|
||||
|
||||
func (h *RuntimeAgentHandler) syncInstanceRuntimeState(ctx context.Context, gateway runtimeAgentGatewayReport, state string) error {
|
||||
func (h *RuntimeAgentHandler) missingGatewayCleanupEligible(binding models.InstanceRuntimeBinding, now time.Time) bool {
|
||||
if h == nil || h.cfg.HeartbeatTimeout <= 0 || binding.LastHealthAt == nil {
|
||||
return false
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(binding.State), services.RuntimeGatewayBindingRunning) {
|
||||
return false
|
||||
}
|
||||
return !binding.LastHealthAt.After(now.Add(-h.cfg.HeartbeatTimeout))
|
||||
}
|
||||
|
||||
func (h *RuntimeAgentHandler) syncInstanceRuntimeState(ctx context.Context, gateway runtimeAgentGatewayReport, lifecycle services.RuntimeGatewayLifecycleState) error {
|
||||
if h.instanceRepo == nil {
|
||||
return nil
|
||||
}
|
||||
instanceState, message := instanceRuntimeStateFromGatewayReport(state, gateway.ErrorMessage)
|
||||
return h.instanceRepo.UpdateRuntimeState(ctx, gateway.InstanceID, instanceState, gateway.Generation, message)
|
||||
}
|
||||
|
||||
func instanceRuntimeStateFromGatewayReport(state string, errorMessage *string) (string, *string) {
|
||||
normalized := strings.ToLower(strings.TrimSpace(state))
|
||||
switch normalized {
|
||||
case "running", "ready", "healthy":
|
||||
return "running", nil
|
||||
case "error", "failed", "failure", "errored":
|
||||
if errorMessage != nil && strings.TrimSpace(*errorMessage) != "" {
|
||||
msg := strings.TrimSpace(*errorMessage)
|
||||
return "error", &msg
|
||||
}
|
||||
msg := "runtime gateway reported " + normalized
|
||||
return "error", &msg
|
||||
case "stopped", "deleted":
|
||||
return "stopped", nil
|
||||
default:
|
||||
msg := "runtime gateway starting"
|
||||
if errorMessage != nil && strings.TrimSpace(*errorMessage) != "" {
|
||||
msg = strings.TrimSpace(*errorMessage)
|
||||
}
|
||||
return "creating", &msg
|
||||
}
|
||||
return h.instanceRepo.UpdateRuntimeState(ctx, gateway.InstanceID, lifecycle.InstanceState, gateway.Generation, lifecycle.Message)
|
||||
}
|
||||
|
||||
func (h *RuntimeAgentHandler) ReportSkills(c *gin.Context) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -211,6 +212,9 @@ func TestRuntimeAgentHandlerGatewayReportOnlyUpdatesCurrentPodBinding(t *testing
|
||||
if bindingRepo.updateRunningCalls != 1 {
|
||||
t.Fatalf("UpdateRunning calls = %d, want 1", bindingRepo.updateRunningCalls)
|
||||
}
|
||||
if len(bindingRepo.deletedInstanceIDs) != 0 {
|
||||
t.Fatalf("stale generation report deleted instances %v", bindingRepo.deletedInstanceIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeAgentHandlerGatewayReportSyncsInstanceRuntimeState(t *testing.T) {
|
||||
@@ -260,6 +264,67 @@ func TestRuntimeAgentHandlerGatewayReportSyncsInstanceRuntimeState(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeAgentHandlerGatewayReportDeletesMissingCurrentPodBinding(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
lastHealthAt := time.Now().UTC().Add(-time.Minute)
|
||||
bindingRepo := &runtimeAgentHandlerBindingRepo{
|
||||
bindings: map[int]*models.InstanceRuntimeBinding{
|
||||
10: {InstanceID: 10, RuntimePodID: 9, Generation: 2, State: "running", LastHealthAt: &lastHealthAt},
|
||||
11: {InstanceID: 11, RuntimePodID: 99, Generation: 2, State: "running"},
|
||||
12: {InstanceID: 12, RuntimePodID: 9, Generation: 2, State: "error"},
|
||||
},
|
||||
}
|
||||
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{})
|
||||
|
||||
router := gin.New()
|
||||
router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/gateways/report", bytes.NewBufferString(`{"pod_id":9,"gateways":[]}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-ClawManager-Agent-Token", "secret")
|
||||
router.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got, want := bindingRepo.deletedInstanceIDs, []int{10}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("deleted instance IDs = %v, want %v", got, want)
|
||||
}
|
||||
if got, want := bindingRepo.deletedRuntimePodIDs, []int64{9}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("released runtime pod IDs = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeAgentHandlerGatewayReportDoesNotDeleteFreshBindingFromFirstEmptySnapshot(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
lastHealthAt := time.Now().UTC()
|
||||
bindingRepo := &runtimeAgentHandlerBindingRepo{
|
||||
bindings: map[int]*models.InstanceRuntimeBinding{
|
||||
10: {InstanceID: 10, RuntimePodID: 9, Generation: 3, State: "running", LastHealthAt: &lastHealthAt},
|
||||
},
|
||||
}
|
||||
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{})
|
||||
|
||||
router := gin.New()
|
||||
router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/gateways/report", bytes.NewBufferString(`{"pod_id":9,"gateways":[]}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-ClawManager-Agent-Token", "secret")
|
||||
router.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(bindingRepo.deletedInstanceIDs) != 0 {
|
||||
t.Fatalf("fresh binding deleted by first empty snapshot: %v", bindingRepo.deletedInstanceIDs)
|
||||
}
|
||||
if bindingRepo.bindings[10] == nil {
|
||||
t.Fatal("fresh binding was removed")
|
||||
}
|
||||
}
|
||||
|
||||
type runtimeAgentHandlerPodRepo struct {
|
||||
updatedPodID int64
|
||||
lastMetrics repository.RuntimePodMetricsUpdate
|
||||
@@ -334,9 +399,11 @@ func (r *runtimeAgentHandlerPodRepo) UpdateMetrics(ctx context.Context, podID in
|
||||
}
|
||||
|
||||
type runtimeAgentHandlerBindingRepo struct {
|
||||
updateRunningCalls int
|
||||
updateStateCalls int
|
||||
bindings map[int]*models.InstanceRuntimeBinding
|
||||
updateRunningCalls int
|
||||
updateStateCalls int
|
||||
bindings map[int]*models.InstanceRuntimeBinding
|
||||
deletedInstanceIDs []int
|
||||
deletedRuntimePodIDs []int64
|
||||
}
|
||||
|
||||
func (r *runtimeAgentHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
|
||||
@@ -352,7 +419,14 @@ func (r *runtimeAgentHandlerBindingRepo) GetRunningByInstanceID(ctx context.Cont
|
||||
}
|
||||
|
||||
func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
|
||||
return nil, nil
|
||||
bindings := make([]models.InstanceRuntimeBinding, 0, len(r.bindings))
|
||||
for _, binding := range r.bindings {
|
||||
if binding.RuntimePodID != runtimePodID {
|
||||
continue
|
||||
}
|
||||
bindings = append(bindings, *binding)
|
||||
}
|
||||
return bindings, nil
|
||||
}
|
||||
|
||||
func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) {
|
||||
@@ -369,14 +443,31 @@ func (r *runtimeAgentHandlerBindingRepo) UpdateState(ctx context.Context, instan
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *runtimeAgentHandlerBindingRepo) DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
|
||||
r.deletedInstanceIDs = append(r.deletedInstanceIDs, instanceID)
|
||||
r.deletedRuntimePodIDs = append(r.deletedRuntimePodIDs, runtimePodID)
|
||||
delete(r.bindings, instanceID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *runtimeAgentHandlerBindingRepo) DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error) {
|
||||
binding := r.bindings[instanceID]
|
||||
if binding == nil || binding.RuntimePodID != runtimePodID || binding.Generation != generation || binding.State != "running" {
|
||||
return false, nil
|
||||
}
|
||||
r.deletedInstanceIDs = append(r.deletedInstanceIDs, instanceID)
|
||||
r.deletedRuntimePodIDs = append(r.deletedRuntimePodIDs, runtimePodID)
|
||||
delete(r.bindings, instanceID)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type runtimeAgentHandlerInstanceRepo struct {
|
||||
statusByID map[int]string
|
||||
generationByID map[int]int
|
||||
|
||||
@@ -429,12 +429,18 @@ func (r *runtimePoolHandlerBindingRepo) UpdateRunning(ctx context.Context, insta
|
||||
func (r *runtimePoolHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
|
||||
return nil
|
||||
}
|
||||
func (r *runtimePoolHandlerBindingRepo) DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
|
||||
return nil
|
||||
}
|
||||
func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
|
||||
return nil
|
||||
}
|
||||
func (r *runtimePoolHandlerBindingRepo) DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type runtimePoolHandlerRolloutRepo struct {
|
||||
created *models.RuntimeRollout
|
||||
|
||||
@@ -2,7 +2,9 @@ package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
@@ -72,19 +74,87 @@ func (r *instanceRepository) GetByID(id int) (*models.Instance, error) {
|
||||
return &instance, nil
|
||||
}
|
||||
|
||||
// GetByAccessToken gets an instance by its lifecycle gateway token.
|
||||
// GetByAccessToken gets an instance by its lifecycle gateway token or a
|
||||
// still-valid historical token alias.
|
||||
func (r *instanceRepository) GetByAccessToken(accessToken string) (*models.Instance, error) {
|
||||
token := strings.TrimSpace(accessToken)
|
||||
if token == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var instance models.Instance
|
||||
err := r.sess.Collection("instances").Find(db.Cond{"access_token": accessToken}).One(&instance)
|
||||
if err != nil {
|
||||
err := r.sess.Collection("instances").Find(db.Cond{"access_token": token}).One(&instance)
|
||||
if err == nil {
|
||||
return &instance, nil
|
||||
}
|
||||
if err != db.ErrNoMoreRows {
|
||||
return nil, fmt.Errorf("failed to get instance by access token: %w", err)
|
||||
}
|
||||
|
||||
return r.getByGatewayTokenAlias(token)
|
||||
}
|
||||
|
||||
func (r *instanceRepository) getByGatewayTokenAlias(accessToken string) (*models.Instance, error) {
|
||||
tokenHash := gatewayTokenHash(accessToken)
|
||||
if tokenHash == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var instance models.Instance
|
||||
now := time.Now().UTC()
|
||||
iter := r.sess.SQL().Iterator(`
|
||||
SELECT i.*
|
||||
FROM instances i
|
||||
JOIN instance_gateway_token_aliases a ON a.instance_id = i.id
|
||||
WHERE a.token_hash = ? AND a.expires_at > ?
|
||||
LIMIT 1
|
||||
`, tokenHash, now)
|
||||
defer iter.Close()
|
||||
if err := iter.One(&instance); err != nil {
|
||||
if err == db.ErrNoMoreRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get instance by access token: %w", err)
|
||||
return nil, fmt.Errorf("failed to get instance by gateway token alias: %w", err)
|
||||
}
|
||||
|
||||
if _, err := r.sess.SQL().Exec(`
|
||||
UPDATE instance_gateway_token_aliases
|
||||
SET last_used_at = ?, updated_at = ?
|
||||
WHERE token_hash = ?
|
||||
`, now, now, tokenHash); err != nil {
|
||||
return nil, fmt.Errorf("failed to mark gateway token alias used: %w", err)
|
||||
}
|
||||
return &instance, nil
|
||||
}
|
||||
|
||||
func (r *instanceRepository) UpsertGatewayTokenAlias(ctx context.Context, instanceID int, accessToken string, expiresAt time.Time) error {
|
||||
tokenHash := gatewayTokenHash(accessToken)
|
||||
if instanceID <= 0 || tokenHash == "" || expiresAt.IsZero() {
|
||||
return nil
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
_, err := r.sess.SQL().ExecContext(ctx, `
|
||||
INSERT INTO instance_gateway_token_aliases (instance_id, token_hash, expires_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
instance_id = VALUES(instance_id),
|
||||
expires_at = GREATEST(expires_at, VALUES(expires_at)),
|
||||
updated_at = VALUES(updated_at)
|
||||
`, instanceID, tokenHash, expiresAt.UTC(), now, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upsert gateway token alias: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func gatewayTokenHash(accessToken string) string {
|
||||
token := strings.TrimSpace(accessToken)
|
||||
if token == "" {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
func (r *instanceRepository) GetByAgentBootstrapToken(bootstrapToken string) (*models.Instance, error) {
|
||||
var instance models.Instance
|
||||
err := r.sess.Collection("instances").Find(db.Cond{"agent_bootstrap_token": bootstrapToken}).One(&instance)
|
||||
|
||||
@@ -53,6 +53,21 @@ func TestBuildV2SchedulerInstanceQueryDefaultsLimit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGatewayTokenHashTrimsAndDoesNotExposeToken(t *testing.T) {
|
||||
hash := gatewayTokenHash(" igt_secret ")
|
||||
if len(hash) != 64 {
|
||||
t.Fatalf("hash length = %d, want 64", len(hash))
|
||||
}
|
||||
if strings.Contains(hash, "igt_secret") {
|
||||
t.Fatalf("hash must not contain raw token")
|
||||
}
|
||||
if hash != gatewayTokenHash("igt_secret") {
|
||||
t.Fatalf("hash should trim surrounding whitespace")
|
||||
}
|
||||
if hash == gatewayTokenHash("igt_other") {
|
||||
t.Fatalf("different tokens should produce different hashes")
|
||||
}
|
||||
}
|
||||
func normalizeSQLForTest(query string) string {
|
||||
return strings.Join(strings.Fields(query), " ")
|
||||
}
|
||||
|
||||
@@ -20,8 +20,10 @@ type InstanceRuntimeBindingRepository interface {
|
||||
ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error)
|
||||
UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error
|
||||
UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error
|
||||
DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error)
|
||||
DeleteByInstanceID(ctx context.Context, instanceID int) error
|
||||
DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error
|
||||
DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error)
|
||||
}
|
||||
|
||||
type instanceRuntimeBindingRepository struct {
|
||||
@@ -152,6 +154,23 @@ func (r *instanceRuntimeBindingRepository) UpdateState(ctx context.Context, inst
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *instanceRuntimeBindingRepository) DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
res, err := r.sess.SQL().ExecContext(ctx, `
|
||||
DELETE FROM instance_runtime_bindings
|
||||
WHERE runtime_pod_id = ? AND gateway_port = ? AND state = 'error'
|
||||
`, runtimePodID, gatewayPort)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to delete stale error instance runtime binding for gateway port: %w", err)
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to inspect stale error instance runtime binding delete: %w", err)
|
||||
}
|
||||
return affected, nil
|
||||
}
|
||||
func (r *instanceRuntimeBindingRepository) getGeneration(ctx context.Context, instanceID int) (int, error) {
|
||||
var currentGeneration int
|
||||
row, err := r.sess.SQL().QueryRowContext(ctx, `
|
||||
@@ -211,3 +230,36 @@ func (r *instanceRuntimeBindingRepository) DeleteByInstanceIDAndReleaseSlot(ctx
|
||||
return nil
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func (r *instanceRuntimeBindingRepository) DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
deleted := false
|
||||
err := r.sess.TxContext(ctx, func(tx db.Session) error {
|
||||
res, err := tx.SQL().ExecContext(ctx, `
|
||||
DELETE FROM instance_runtime_bindings
|
||||
WHERE instance_id = ? AND runtime_pod_id = ? AND generation = ? AND state = 'running'
|
||||
`, instanceID, runtimePodID, generation)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete current running instance runtime binding: %w", err)
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to inspect current running instance runtime binding delete: %w", err)
|
||||
}
|
||||
if affected == 0 {
|
||||
return nil
|
||||
}
|
||||
if _, err := tx.SQL().ExecContext(ctx, `
|
||||
UPDATE runtime_pods
|
||||
SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ?
|
||||
WHERE id = ?
|
||||
`, time.Now().UTC(), runtimePodID); err != nil {
|
||||
return fmt.Errorf("failed to release runtime pod slot: %w", err)
|
||||
}
|
||||
deleted = true
|
||||
return nil
|
||||
}, nil)
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -111,6 +112,19 @@ func (s *InstanceAccessService) ValidateToken(token string) (*AccessToken, error
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (s *InstanceAccessService) IsInstanceAccessToken(token string) bool {
|
||||
value := strings.TrimSpace(token)
|
||||
if value == "" || strings.Count(value, ".") != 2 {
|
||||
return false
|
||||
}
|
||||
|
||||
parsed, _, err := jwt.NewParser().ParseUnverified(value, &instanceAccessClaims{})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
claims, ok := parsed.Claims.(*instanceAccessClaims)
|
||||
return ok && claims.TokenType == "instance_access" && strings.TrimSpace(claims.AccessURL) != ""
|
||||
}
|
||||
func (s *InstanceAccessService) validateSignedToken(token string) (*AccessToken, error) {
|
||||
parsed, err := jwt.ParseWithClaims(token, &instanceAccessClaims{}, func(token *jwt.Token) (interface{}, error) {
|
||||
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
|
||||
|
||||
@@ -37,9 +37,12 @@ func TestBuildInstancePodEnvAppliesOverridesAfterDefaults(t *testing.T) {
|
||||
t.Setenv("K8S_NAMESPACE", "")
|
||||
|
||||
raw, err := marshalEnvironmentOverrides(map[string]string{
|
||||
"SUBFOLDER": "/custom-proxy",
|
||||
"KASM_SVC_ACCEPT_CUT_TEXT": "-AcceptCutText 1",
|
||||
"CUSTOM": "enabled",
|
||||
"SUBFOLDER": "/custom-proxy",
|
||||
"KASM_SVC_ACCEPT_CUT_TEXT": "-AcceptCutText 1",
|
||||
"SELKIES_CLIPBOARD_ENABLED": "true|locked",
|
||||
"SELKIES_CLIPBOARD_IN_ENABLED": "true|locked",
|
||||
"SELKIES_CLIPBOARD_OUT_ENABLED": "true|locked",
|
||||
"CUSTOM": "enabled",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshalEnvironmentOverrides returned error: %v", err)
|
||||
@@ -51,10 +54,7 @@ func TestBuildInstancePodEnvAppliesOverridesAfterDefaults(t *testing.T) {
|
||||
EnvironmentOverridesJSON: raw,
|
||||
}
|
||||
|
||||
env, err := buildInstancePodEnv(instance, map[string]string{
|
||||
"TITLE": "ClawManager Webtop",
|
||||
"SUBFOLDER": "/",
|
||||
}, nil, nil)
|
||||
env, err := buildInstancePodEnv(instance, defaultWebtopDesktopEnv("ClawManager Webtop"), nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("buildInstancePodEnv returned error: %v", err)
|
||||
}
|
||||
@@ -65,6 +65,15 @@ func TestBuildInstancePodEnvAppliesOverridesAfterDefaults(t *testing.T) {
|
||||
if env["KASM_SVC_ACCEPT_CUT_TEXT"] != "-AcceptCutText 1" {
|
||||
t.Fatalf("expected clipboard policy override to win, got %q", env["KASM_SVC_ACCEPT_CUT_TEXT"])
|
||||
}
|
||||
for _, key := range []string{
|
||||
"SELKIES_CLIPBOARD_ENABLED",
|
||||
"SELKIES_CLIPBOARD_IN_ENABLED",
|
||||
"SELKIES_CLIPBOARD_OUT_ENABLED",
|
||||
} {
|
||||
if got := env[key]; got != "true|locked" {
|
||||
t.Fatalf("expected %s override to win, got %q", key, got)
|
||||
}
|
||||
}
|
||||
if env["CUSTOM"] != "enabled" {
|
||||
t.Fatalf("expected custom environment variable to be merged")
|
||||
}
|
||||
|
||||
@@ -144,9 +144,11 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
|
||||
return err
|
||||
}
|
||||
|
||||
// Copy query parameters (excluding token)
|
||||
managedGatewayToken := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType)
|
||||
|
||||
// Copy query parameters, excluding ClawManager-owned proxy/gateway tokens.
|
||||
queryParams := r.URL.Query()
|
||||
removeProxyAccessTokenQuery(queryParams, token)
|
||||
s.removeProxyAccessTokenQuery(queryParams, token, managedGatewayToken)
|
||||
if len(queryParams) > 0 {
|
||||
targetURL.RawQuery = queryParams.Encode()
|
||||
}
|
||||
@@ -172,9 +174,7 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
|
||||
proxyReq.Header.Set("X-Forwarded-Host", r.Host)
|
||||
proxyReq.Header.Set("X-Forwarded-Proto", requestScheme(r))
|
||||
proxyReq.Header.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID))
|
||||
if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" {
|
||||
proxyReq.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
setManagedRuntimeGatewayAuthHeaders(proxyReq.Header, managedGatewayToken)
|
||||
if shouldRewriteHTML {
|
||||
proxyReq.Header.Del("Accept-Encoding")
|
||||
}
|
||||
@@ -270,9 +270,11 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in
|
||||
return err
|
||||
}
|
||||
|
||||
// Copy query parameters (excluding token)
|
||||
managedGatewayToken := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType)
|
||||
|
||||
// Copy query parameters, excluding ClawManager-owned proxy/gateway tokens.
|
||||
queryParams := r.URL.Query()
|
||||
removeProxyAccessTokenQuery(queryParams, token)
|
||||
s.removeProxyAccessTokenQuery(queryParams, token, managedGatewayToken)
|
||||
if len(queryParams) > 0 {
|
||||
targetURL.RawQuery = queryParams.Encode()
|
||||
}
|
||||
@@ -293,8 +295,8 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in
|
||||
upstreamHeader.Set("X-Forwarded-Host", r.Host)
|
||||
upstreamHeader.Set("X-Forwarded-Proto", requestScheme(r))
|
||||
upstreamHeader.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID))
|
||||
if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" {
|
||||
upstreamHeader.Set("Authorization", "Bearer "+token)
|
||||
setManagedRuntimeGatewayAuthHeaders(upstreamHeader, managedGatewayToken)
|
||||
if managedGatewayToken != "" {
|
||||
upstreamHeader.Set("Origin", s.openClawWebSocketOrigin(targetURL))
|
||||
}
|
||||
|
||||
@@ -389,6 +391,18 @@ func (s *InstanceProxyService) removeHopByHopHeaders(header http.Header) {
|
||||
}
|
||||
}
|
||||
|
||||
func setManagedRuntimeGatewayAuthHeaders(header http.Header, token string) {
|
||||
token = strings.TrimSpace(token)
|
||||
if token == "" {
|
||||
return
|
||||
}
|
||||
header.Set("Authorization", "Bearer "+token)
|
||||
header.Set("X-Api-Key", token)
|
||||
header.Set("X-OpenAI-Api-Key", token)
|
||||
header.Set("OpenAI-Api-Key", token)
|
||||
header.Set("X-ClawManager-Instance-Token", token)
|
||||
header.Set("X-ClawManager-LLM-API-Key", token)
|
||||
}
|
||||
func (s *InstanceProxyService) managedRuntimeGatewayBearerToken(ctx context.Context, instanceID int, instanceType string) string {
|
||||
if s == nil || s.instanceRepo == nil {
|
||||
return ""
|
||||
@@ -838,14 +852,14 @@ func proxyURLWithPath(instanceID int, targetPath, token string) string {
|
||||
return fmt.Sprintf("%s?token=%s", raw, url.QueryEscape(token))
|
||||
}
|
||||
|
||||
func removeProxyAccessTokenQuery(query url.Values, accessToken string) {
|
||||
func (s *InstanceProxyService) removeProxyAccessTokenQuery(query url.Values, accessToken, managedGatewayToken string) {
|
||||
values := query["token"]
|
||||
if len(values) == 0 {
|
||||
return
|
||||
}
|
||||
filtered := values[:0]
|
||||
for _, value := range values {
|
||||
if value != accessToken {
|
||||
if !s.shouldRemoveProxyTokenQueryValue(value, accessToken, managedGatewayToken) {
|
||||
filtered = append(filtered, value)
|
||||
}
|
||||
}
|
||||
@@ -856,6 +870,23 @@ func removeProxyAccessTokenQuery(query url.Values, accessToken string) {
|
||||
query["token"] = filtered
|
||||
}
|
||||
|
||||
func (s *InstanceProxyService) shouldRemoveProxyTokenQueryValue(value, accessToken, managedGatewayToken string) bool {
|
||||
candidate := strings.TrimSpace(value)
|
||||
if candidate == "" {
|
||||
return false
|
||||
}
|
||||
if candidate == accessToken || (managedGatewayToken != "" && candidate == managedGatewayToken) {
|
||||
return true
|
||||
}
|
||||
if s != nil && s.accessService != nil && s.accessService.IsInstanceAccessToken(candidate) {
|
||||
return true
|
||||
}
|
||||
return managedGatewayToken != "" && looksLikeManagedInstanceGatewayToken(candidate)
|
||||
}
|
||||
|
||||
func looksLikeManagedInstanceGatewayToken(value string) bool {
|
||||
return strings.HasPrefix(strings.TrimSpace(value), "igt_")
|
||||
}
|
||||
func proxyBaseForRequestPath(requestPath string, instanceID int) string {
|
||||
prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
|
||||
path := strings.TrimSpace(requestPath)
|
||||
|
||||
@@ -94,6 +94,12 @@ func TestInstanceProxyServiceInjectsInstanceTokenForHermesLite(t *testing.T) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
|
||||
t.Fatalf("Authorization = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("X-Api-Key"); got != instanceToken {
|
||||
t.Fatalf("X-Api-Key = %q", got)
|
||||
}
|
||||
if got := r.Header.Get("X-ClawManager-Instance-Token"); got != instanceToken {
|
||||
t.Fatalf("X-ClawManager-Instance-Token = %q", got)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
@@ -139,6 +145,7 @@ func TestInstanceProxyServiceInjectsInstanceTokenForHermesLite(t *testing.T) {
|
||||
service.httpClient = upstream.Client()
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/127/proxy/chat?token="+url.QueryEscape(token.Token), nil)
|
||||
req.Header.Set("X-Api-Key", "stale-session-key")
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
if err := service.ProxyRequest(req.Context(), 127, token.Token, rec, req); err != nil {
|
||||
@@ -284,6 +291,94 @@ func TestInstanceProxyServicePreservesHermesRuntimeQueryToken(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstanceProxyServiceStripsStaleHermesGatewayQueryToken(t *testing.T) {
|
||||
instanceToken := "igt_hermes_instance"
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/ws" {
|
||||
t.Fatalf("unexpected upstream path %s", r.URL.Path)
|
||||
}
|
||||
if got := r.URL.RawQuery; got != "" {
|
||||
t.Fatalf("upstream query = %q, want empty", got)
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
|
||||
t.Fatalf("Authorization = %q", got)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
|
||||
workspacePath := "/workspaces/hermes/user-45/instance-132"
|
||||
instanceRepo := newV2LifecycleInstanceRepo()
|
||||
instanceRepo.byID[132] = &models.Instance{
|
||||
ID: 132,
|
||||
UserID: 45,
|
||||
Type: "hermes",
|
||||
RuntimeType: "gateway",
|
||||
InstanceMode: InstanceModeLite,
|
||||
Status: "running",
|
||||
AccessToken: &instanceToken,
|
||||
WorkspacePath: &workspacePath,
|
||||
RuntimeGeneration: 5,
|
||||
}
|
||||
bindingRepo := newFakeRuntimeBindingRepo()
|
||||
bindingRepo.bindings[132] = &models.InstanceRuntimeBinding{
|
||||
InstanceID: 132,
|
||||
RuntimePodID: 15,
|
||||
GatewayPort: gatewayPort,
|
||||
State: "running",
|
||||
Generation: 5,
|
||||
}
|
||||
podRepo := &fakeRuntimePodRepo{
|
||||
pods: map[int64]*models.RuntimePod{
|
||||
15: {ID: 15, PodIP: &podIP, State: "ready"},
|
||||
},
|
||||
}
|
||||
accessService := NewInstanceAccessService()
|
||||
defer accessService.Stop()
|
||||
accessToken, err := accessService.GenerateToken(45, 132, "hermes", "/api/v1/instances/132/proxy/chat/", "", 3000, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateToken returned error: %v", err)
|
||||
}
|
||||
service := NewInstanceProxyService(accessService)
|
||||
service.instanceRepo = instanceRepo
|
||||
service.bindingRepo = bindingRepo
|
||||
service.runtimePodRepo = podRepo
|
||||
service.httpClient = upstream.Client()
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/132/proxy/api/ws?token=igt_stale_gateway", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
if err := service.ProxyRequest(req.Context(), 132, accessToken.Token, rec, req); err != nil {
|
||||
t.Fatalf("ProxyRequest returned error: %v", err)
|
||||
}
|
||||
if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
|
||||
t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstanceProxyServiceStripsStaleProxyAccessTokenQuery(t *testing.T) {
|
||||
accessService := NewInstanceAccessService()
|
||||
defer accessService.Stop()
|
||||
currentToken, err := accessService.GenerateToken(45, 133, "hermes", "/api/v1/instances/133/proxy/chat/", "", 3000, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateToken current returned error: %v", err)
|
||||
}
|
||||
oldToken, err := accessService.GenerateToken(45, 133, "hermes", "/api/v1/instances/133/proxy/chat/", "", 3000, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateToken old returned error: %v", err)
|
||||
}
|
||||
service := NewInstanceProxyService(accessService)
|
||||
query := url.Values{"token": []string{oldToken.Token, "hermes-session-token"}}
|
||||
|
||||
service.removeProxyAccessTokenQuery(query, currentToken.Token, "igt_hermes_instance")
|
||||
|
||||
got := query["token"]
|
||||
if len(got) != 1 || got[0] != "hermes-session-token" {
|
||||
t.Fatalf("token query = %#v, want only Hermes session token", got)
|
||||
}
|
||||
}
|
||||
func TestInstanceProxyServiceRewritesHermesLiteHTMLBase(t *testing.T) {
|
||||
instanceToken := "igt_hermes_instance"
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -17,8 +17,9 @@ type InstanceRuntimeConfig struct {
|
||||
}
|
||||
|
||||
const (
|
||||
kasmClipboardSendDisabled = "-SendCutText 0"
|
||||
kasmClipboardAcceptDisabled = "-AcceptCutText 0"
|
||||
kasmClipboardSendEnabled = "-SendCutText 1"
|
||||
kasmClipboardAcceptEnabled = "-AcceptCutText 1"
|
||||
selkiesClipboardEnabled = "true|locked"
|
||||
)
|
||||
|
||||
func buildRuntimeConfig(instanceType, osType, osVersion string, registry, tag *string) InstanceRuntimeConfig {
|
||||
@@ -117,10 +118,13 @@ func defaultEnvForInstanceType(instanceType string) map[string]string {
|
||||
|
||||
func defaultWebtopDesktopEnv(title string) map[string]string {
|
||||
return map[string]string{
|
||||
"TITLE": title,
|
||||
"SUBFOLDER": "/",
|
||||
"KASM_SVC_SEND_CUT_TEXT": kasmClipboardSendDisabled,
|
||||
"KASM_SVC_ACCEPT_CUT_TEXT": kasmClipboardAcceptDisabled,
|
||||
"TITLE": title,
|
||||
"SUBFOLDER": "/",
|
||||
"KASM_SVC_SEND_CUT_TEXT": kasmClipboardSendEnabled,
|
||||
"KASM_SVC_ACCEPT_CUT_TEXT": kasmClipboardAcceptEnabled,
|
||||
"SELKIES_CLIPBOARD_ENABLED": selkiesClipboardEnabled,
|
||||
"SELKIES_CLIPBOARD_IN_ENABLED": selkiesClipboardEnabled,
|
||||
"SELKIES_CLIPBOARD_OUT_ENABLED": selkiesClipboardEnabled,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -37,24 +37,39 @@ func TestBuildRuntimeConfig_HermesUsesWebtopDefaults(t *testing.T) {
|
||||
if config.Env["SUBFOLDER"] != "/" {
|
||||
t.Fatalf("expected Hermes default SUBFOLDER /, got %q", config.Env["SUBFOLDER"])
|
||||
}
|
||||
if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendDisabled {
|
||||
t.Fatalf("expected Hermes to disable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"])
|
||||
if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendEnabled {
|
||||
t.Fatalf("expected Hermes to enable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"])
|
||||
}
|
||||
if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptDisabled {
|
||||
t.Fatalf("expected Hermes to disable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"])
|
||||
if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptEnabled {
|
||||
t.Fatalf("expected Hermes to enable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"])
|
||||
}
|
||||
assertSelkiesClipboardEnabled(t, config.Env)
|
||||
if !usesWebtopImage("hermes") {
|
||||
t.Fatalf("expected Hermes to use webtop proxy behavior")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRuntimeConfig_OpenClawDisablesKasmClipboardSync(t *testing.T) {
|
||||
func TestBuildRuntimeConfig_OpenClawEnablesDesktopClipboardSync(t *testing.T) {
|
||||
config := buildRuntimeConfig("openclaw", "openclaw", "latest", nil, nil)
|
||||
|
||||
if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendDisabled {
|
||||
t.Fatalf("expected OpenClaw to disable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"])
|
||||
if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendEnabled {
|
||||
t.Fatalf("expected OpenClaw to enable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"])
|
||||
}
|
||||
if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptDisabled {
|
||||
t.Fatalf("expected OpenClaw to disable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"])
|
||||
if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptEnabled {
|
||||
t.Fatalf("expected OpenClaw to enable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"])
|
||||
}
|
||||
assertSelkiesClipboardEnabled(t, config.Env)
|
||||
}
|
||||
|
||||
func assertSelkiesClipboardEnabled(t *testing.T, env map[string]string) {
|
||||
t.Helper()
|
||||
for _, key := range []string{
|
||||
"SELKIES_CLIPBOARD_ENABLED",
|
||||
"SELKIES_CLIPBOARD_IN_ENABLED",
|
||||
"SELKIES_CLIPBOARD_OUT_ENABLED",
|
||||
} {
|
||||
if got := env[key]; got != selkiesClipboardEnabled {
|
||||
t.Fatalf("expected %s=%q, got %q", key, selkiesClipboardEnabled, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,6 +220,14 @@ type instanceService struct {
|
||||
networkPolicyService *k8s.NetworkPolicyService
|
||||
}
|
||||
|
||||
const (
|
||||
defaultGatewayTokenAliasTTL = 7 * 24 * time.Hour
|
||||
gatewayTokenAliasTTLEnv = "CLAWMANAGER_GATEWAY_TOKEN_ALIAS_TTL_HOURS"
|
||||
)
|
||||
|
||||
type gatewayTokenAliasRecorder interface {
|
||||
UpsertGatewayTokenAlias(ctx context.Context, instanceID int, accessToken string, expiresAt time.Time) error
|
||||
}
|
||||
type gatewayModelInjection struct {
|
||||
defaultModel string
|
||||
modelsJSON string
|
||||
@@ -987,7 +995,9 @@ func (s *instanceService) securityModeForInstance(instanceType string) k8s.PodSe
|
||||
|
||||
func (s *instanceService) ensureGatewayToken(instance *models.Instance) (string, error) {
|
||||
if instance.AccessToken != nil && strings.TrimSpace(*instance.AccessToken) != "" {
|
||||
return strings.TrimSpace(*instance.AccessToken), nil
|
||||
token := strings.TrimSpace(*instance.AccessToken)
|
||||
s.refreshGatewayTokenAlias(instance.ID, token)
|
||||
return token, nil
|
||||
}
|
||||
|
||||
tokenBytes := make([]byte, 32)
|
||||
@@ -1002,9 +1012,34 @@ func (s *instanceService) ensureGatewayToken(instance *models.Instance) (string,
|
||||
return "", fmt.Errorf("failed to persist instance gateway token: %w", err)
|
||||
}
|
||||
|
||||
s.refreshGatewayTokenAlias(instance.ID, token)
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (s *instanceService) refreshGatewayTokenAlias(instanceID int, token string) {
|
||||
recorder, ok := s.instanceRepo.(gatewayTokenAliasRecorder)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
ttl := gatewayTokenAliasTTL()
|
||||
if ttl <= 0 || strings.TrimSpace(token) == "" || instanceID <= 0 {
|
||||
return
|
||||
}
|
||||
if err := recorder.UpsertGatewayTokenAlias(context.Background(), instanceID, token, time.Now().UTC().Add(ttl)); err != nil {
|
||||
fmt.Printf("Warning: failed to refresh gateway token alias for instance %d: %v\n", instanceID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func gatewayTokenAliasTTL() time.Duration {
|
||||
value := optionalIntEnv(gatewayTokenAliasTTLEnv)
|
||||
if value == nil {
|
||||
return defaultGatewayTokenAliasTTL
|
||||
}
|
||||
if *value <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(*value) * time.Hour
|
||||
}
|
||||
func (s *instanceService) buildGatewayEnv(instance *models.Instance) (map[string]string, error) {
|
||||
if instance == nil || instance.AccessToken == nil || strings.TrimSpace(*instance.AccessToken) == "" {
|
||||
return map[string]string{}, nil
|
||||
@@ -1024,6 +1059,7 @@ func (s *instanceService) buildGatewayEnv(instance *models.Instance) (map[string
|
||||
}
|
||||
|
||||
token := strings.TrimSpace(*instance.AccessToken)
|
||||
s.refreshGatewayTokenAlias(instance.ID, token)
|
||||
return map[string]string{
|
||||
"CLAWMANAGER_LLM_BASE_URL": baseURL,
|
||||
"CLAWMANAGER_LLM_API_KEY": token,
|
||||
|
||||
@@ -1,12 +1,31 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"clawreef/internal/models"
|
||||
)
|
||||
|
||||
func TestGatewayTokenAliasTTLConfig(t *testing.T) {
|
||||
t.Setenv(gatewayTokenAliasTTLEnv, "")
|
||||
if got := gatewayTokenAliasTTL(); got != defaultGatewayTokenAliasTTL {
|
||||
t.Fatalf("default gateway token alias TTL = %v, want %v", got, defaultGatewayTokenAliasTTL)
|
||||
}
|
||||
|
||||
t.Setenv(gatewayTokenAliasTTLEnv, "2")
|
||||
if got := gatewayTokenAliasTTL(); got != 2*time.Hour {
|
||||
t.Fatalf("configured gateway token alias TTL = %v, want 2h", got)
|
||||
}
|
||||
|
||||
t.Setenv(gatewayTokenAliasTTLEnv, "0")
|
||||
if got := gatewayTokenAliasTTL(); got != 0 {
|
||||
t.Fatalf("disabled gateway token alias TTL = %v, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
type stubLLMModelRepository struct {
|
||||
active []models.LLMModel
|
||||
}
|
||||
@@ -109,8 +128,50 @@ func TestBuildGatewayEnvEnsuresMissingGatewayToken(t *testing.T) {
|
||||
if got := instanceRepo.updated[68]; got == nil || got.AccessToken == nil || *got.AccessToken != *instance.AccessToken {
|
||||
t.Fatalf("repository update did not persist provisioned token: %#v", instanceRepo.updated[68])
|
||||
}
|
||||
alias, ok := instanceRepo.aliases[68]
|
||||
if !ok {
|
||||
t.Fatal("repository did not record provisioned token as a gateway alias")
|
||||
}
|
||||
if alias.token != *instance.AccessToken {
|
||||
t.Fatalf("gateway alias token = %q, want provisioned token %q", alias.token, *instance.AccessToken)
|
||||
}
|
||||
if alias.expiresAt.Before(time.Now().Add(6 * 24 * time.Hour)) {
|
||||
t.Fatalf("gateway alias expiry = %v, want roughly the default compatibility window", alias.expiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildGatewayEnvRecordsExistingGatewayTokenAlias(t *testing.T) {
|
||||
t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm")
|
||||
|
||||
token := "igt_existing_token"
|
||||
instanceRepo := &stubGatewayEnvInstanceRepository{fakeRuntimeInstanceRepo: newFakeRuntimeInstanceRepo()}
|
||||
service := &instanceService{
|
||||
instanceRepo: instanceRepo,
|
||||
llmModelRepo: &stubLLMModelRepository{
|
||||
active: []models.LLMModel{{DisplayName: "auto"}},
|
||||
},
|
||||
}
|
||||
|
||||
env, err := service.BuildGatewayEnv(&models.Instance{
|
||||
ID: 69,
|
||||
UserID: 1,
|
||||
Type: "hermes",
|
||||
AccessToken: &token,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("BuildGatewayEnv returned error: %v", err)
|
||||
}
|
||||
if env["CLAWMANAGER_LLM_API_KEY"] != token {
|
||||
t.Fatalf("CLAWMANAGER_LLM_API_KEY = %q, want existing token", env["CLAWMANAGER_LLM_API_KEY"])
|
||||
}
|
||||
alias, ok := instanceRepo.aliases[69]
|
||||
if !ok {
|
||||
t.Fatal("repository did not record existing token as a gateway alias")
|
||||
}
|
||||
if alias.token != token {
|
||||
t.Fatalf("gateway alias token = %q, want existing token %q", alias.token, token)
|
||||
}
|
||||
}
|
||||
func TestBuildGatewayEnvMergesEnvironmentOverrides(t *testing.T) {
|
||||
t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm")
|
||||
token := "igt_test_token"
|
||||
@@ -169,9 +230,15 @@ func TestBuildGatewayEnvSkipsUnmanagedRuntime(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type gatewayTokenAliasRecord struct {
|
||||
token string
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
type stubGatewayEnvInstanceRepository struct {
|
||||
*fakeRuntimeInstanceRepo
|
||||
updated map[int]*models.Instance
|
||||
aliases map[int]gatewayTokenAliasRecord
|
||||
}
|
||||
|
||||
func (r *stubGatewayEnvInstanceRepository) Update(instance *models.Instance) error {
|
||||
@@ -183,6 +250,13 @@ func (r *stubGatewayEnvInstanceRepository) Update(instance *models.Instance) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *stubGatewayEnvInstanceRepository) UpsertGatewayTokenAlias(ctx context.Context, instanceID int, accessToken string, expiresAt time.Time) error {
|
||||
if r.aliases == nil {
|
||||
r.aliases = map[int]gatewayTokenAliasRecord{}
|
||||
}
|
||||
r.aliases[instanceID] = gatewayTokenAliasRecord{token: accessToken, expiresAt: expiresAt}
|
||||
return nil
|
||||
}
|
||||
func TestBuildAgentEnvInjectsHermesAgentConfig(t *testing.T) {
|
||||
t.Setenv("CLAWMANAGER_AGENT_CONTROL_BASE_URL", "http://agent-control.example")
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ const (
|
||||
runtimeWorkspaceVolume = "workspaces"
|
||||
defaultWorkspaceMount = "/workspaces"
|
||||
defaultGatewayPortStart = 20000
|
||||
defaultGatewayPortEnd = 20099
|
||||
defaultGatewayPortEnd = 20299
|
||||
hermesTUIDir = "/usr/local/lib/hermes-agent/ui-tui"
|
||||
)
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ func TestBuildRuntimeDeployment(t *testing.T) {
|
||||
requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "openclaw")
|
||||
requireEnv(t, container, "CLAWMANAGER_AGENT_PORT", "19090")
|
||||
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "20000")
|
||||
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "20099")
|
||||
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "20299")
|
||||
requireEnv(t, container, "CLAWMANAGER_AGENT_CONTROL_TOKEN", "control-secret")
|
||||
requireEnv(t, container, "CLAWMANAGER_AGENT_REPORT_TOKEN", "report-secret")
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ func TestRuntimeAgentClientCreateGateway(t *testing.T) {
|
||||
UserID: 8,
|
||||
AgentType: "openclaw",
|
||||
WorkspacePath: "/workspaces/openclaw/user-8/instance-7",
|
||||
PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099},
|
||||
PortRange: RuntimeAgentPortRange{Start: RuntimeGatewayPortStart, End: RuntimeGatewayPortEnd},
|
||||
UID: 200007,
|
||||
GID: 200007,
|
||||
CPUCores: 2,
|
||||
|
||||
@@ -18,9 +18,12 @@ const (
|
||||
RuntimeBackendShell = "shell"
|
||||
|
||||
RuntimeGatewayPortStart = 20000
|
||||
RuntimeGatewayPortEnd = 20099
|
||||
RuntimePodCapacity = 100
|
||||
RuntimeLinuxIDBase = 200000
|
||||
// OpenClaw gateway slots reserve adjacent service ports in addition to the
|
||||
// public gateway port. Capacity is instance count, not raw port count.
|
||||
RuntimeGatewayPortsPerInstance = 3
|
||||
RuntimeGatewayPortEnd = RuntimeGatewayPortStart + RuntimePodCapacity*RuntimeGatewayPortsPerInstance - 1
|
||||
RuntimeLinuxIDBase = 200000
|
||||
)
|
||||
|
||||
func NormalizeV2RuntimeType(instanceType string) (string, bool) {
|
||||
|
||||
@@ -36,6 +36,19 @@ func TestRuntimeLinuxID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeGatewayPortRangeMatchesInstanceCapacity(t *testing.T) {
|
||||
if got, want := RuntimeGatewayPortsPerInstance, 3; got != want {
|
||||
t.Fatalf("expected %d ports per gateway instance, got %d", want, got)
|
||||
}
|
||||
wantEnd := RuntimeGatewayPortStart + RuntimePodCapacity*RuntimeGatewayPortsPerInstance - 1
|
||||
if RuntimeGatewayPortEnd != wantEnd {
|
||||
t.Fatalf("gateway port end = %d, want %d", RuntimeGatewayPortEnd, wantEnd)
|
||||
}
|
||||
if got := (RuntimeGatewayPortEnd - RuntimeGatewayPortStart + 1) / RuntimeGatewayPortsPerInstance; got != RuntimePodCapacity {
|
||||
t.Fatalf("gateway port range supports %d instances, want %d", got, RuntimePodCapacity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstanceModeRuntimeTypeMapping(t *testing.T) {
|
||||
if got, ok := RuntimeTypeForInstanceMode(" lite "); !ok || got != RuntimeBackendGateway {
|
||||
t.Fatalf("lite runtime type = %q/%v, want gateway/true", got, ok)
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package services
|
||||
|
||||
import "strings"
|
||||
|
||||
const (
|
||||
RuntimeGatewayBindingCreating = "creating"
|
||||
RuntimeGatewayBindingRunning = "running"
|
||||
RuntimeGatewayBindingError = "error"
|
||||
RuntimeGatewayBindingStopped = "stopped"
|
||||
)
|
||||
|
||||
type RuntimeGatewayLifecycleState struct {
|
||||
RawState string
|
||||
BindingState string
|
||||
InstanceState string
|
||||
Running bool
|
||||
Recognized bool
|
||||
EventType string
|
||||
Message *string
|
||||
}
|
||||
|
||||
func NormalizeRuntimeGatewayLifecycle(rawState string, reportedMessage *string) RuntimeGatewayLifecycleState {
|
||||
raw := strings.ToLower(strings.TrimSpace(rawState))
|
||||
message := normalizedRuntimeGatewayMessage(reportedMessage)
|
||||
state := RuntimeGatewayLifecycleState{RawState: raw, Recognized: true}
|
||||
|
||||
switch raw {
|
||||
case "running", "ready", "healthy":
|
||||
state.BindingState = RuntimeGatewayBindingRunning
|
||||
state.InstanceState = "running"
|
||||
state.Running = true
|
||||
state.EventType = "runtime.instance.running"
|
||||
case "starting", "creating", "pending":
|
||||
state.BindingState = RuntimeGatewayBindingCreating
|
||||
state.InstanceState = "creating"
|
||||
state.EventType = "runtime.instance.starting"
|
||||
state.Message = messageOrDefault(message, "runtime gateway starting")
|
||||
case "error", "failed", "failure", "errored":
|
||||
state.BindingState = RuntimeGatewayBindingError
|
||||
state.InstanceState = "error"
|
||||
state.EventType = "runtime.instance.error"
|
||||
state.Message = messageOrDefault(message, "runtime gateway reported "+raw)
|
||||
case "stopped", "deleted":
|
||||
state.BindingState = RuntimeGatewayBindingStopped
|
||||
state.InstanceState = "stopped"
|
||||
state.EventType = "runtime.instance.stopped"
|
||||
state.Message = message
|
||||
case "":
|
||||
state.Recognized = false
|
||||
state.BindingState = RuntimeGatewayBindingCreating
|
||||
state.InstanceState = "creating"
|
||||
state.EventType = "runtime.instance.starting"
|
||||
state.Message = messageOrDefault(message, "runtime gateway returned an empty state")
|
||||
default:
|
||||
state.Recognized = false
|
||||
state.BindingState = RuntimeGatewayBindingCreating
|
||||
state.InstanceState = "creating"
|
||||
state.EventType = "runtime.instance.starting"
|
||||
state.Message = messageOrDefault(message, "runtime gateway reported unrecognized state: "+raw)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func (state RuntimeGatewayLifecycleState) CreateAccepted() bool {
|
||||
if !state.Recognized {
|
||||
return false
|
||||
}
|
||||
return state.BindingState == RuntimeGatewayBindingRunning || state.BindingState == RuntimeGatewayBindingCreating
|
||||
}
|
||||
|
||||
func normalizedRuntimeGatewayMessage(message *string) *string {
|
||||
if message == nil {
|
||||
return nil
|
||||
}
|
||||
trimmed := strings.TrimSpace(*message)
|
||||
if trimmed == "" {
|
||||
return nil
|
||||
}
|
||||
return &trimmed
|
||||
}
|
||||
|
||||
func messageOrDefault(message *string, fallback string) *string {
|
||||
if message != nil {
|
||||
return message
|
||||
}
|
||||
return &fallback
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package services
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeRuntimeGatewayLifecycle(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
bindingState string
|
||||
instanceState string
|
||||
running bool
|
||||
recognized bool
|
||||
createAccepted bool
|
||||
}{
|
||||
{name: "running", raw: "running", bindingState: "running", instanceState: "running", running: true, recognized: true, createAccepted: true},
|
||||
{name: "ready", raw: "ready", bindingState: "running", instanceState: "running", running: true, recognized: true, createAccepted: true},
|
||||
{name: "healthy", raw: "healthy", bindingState: "running", instanceState: "running", running: true, recognized: true, createAccepted: true},
|
||||
{name: "starting", raw: "starting", bindingState: "creating", instanceState: "creating", recognized: true, createAccepted: true},
|
||||
{name: "creating", raw: "creating", bindingState: "creating", instanceState: "creating", recognized: true, createAccepted: true},
|
||||
{name: "pending", raw: "pending", bindingState: "creating", instanceState: "creating", recognized: true, createAccepted: true},
|
||||
{name: "error", raw: "error", bindingState: "error", instanceState: "error", recognized: true},
|
||||
{name: "stopped", raw: "stopped", bindingState: "stopped", instanceState: "stopped", recognized: true},
|
||||
{name: "empty", raw: "", bindingState: "creating", instanceState: "creating"},
|
||||
{name: "unknown", raw: "mystery", bindingState: "creating", instanceState: "creating"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := NormalizeRuntimeGatewayLifecycle(tt.raw, nil)
|
||||
if got.BindingState != tt.bindingState || got.InstanceState != tt.instanceState || got.Running != tt.running || got.Recognized != tt.recognized {
|
||||
t.Fatalf("state = %+v", got)
|
||||
}
|
||||
if got.CreateAccepted() != tt.createAccepted {
|
||||
t.Fatalf("CreateAccepted = %t, want %t", got.CreateAccepted(), tt.createAccepted)
|
||||
}
|
||||
if !tt.running && got.Message == nil && tt.instanceState != "stopped" {
|
||||
t.Fatal("non-running state is missing a diagnostic message")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRuntimeGatewayLifecyclePreservesReportedError(t *testing.T) {
|
||||
message := "gateway port is not listening"
|
||||
got := NormalizeRuntimeGatewayLifecycle("failed", &message)
|
||||
if got.Message == nil || *got.Message != message {
|
||||
t.Fatalf("message = %#v, want %q", got.Message, message)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"clawreef/internal/models"
|
||||
@@ -32,11 +33,21 @@ type RuntimeScheduler struct {
|
||||
gatewayPortEnd int
|
||||
heartbeatTimeout time.Duration
|
||||
maxGatewaysPerPod int
|
||||
|
||||
gatewayCreateLocksMu sync.Mutex
|
||||
gatewayCreateLocks map[int64]*sync.Mutex
|
||||
}
|
||||
|
||||
var errRuntimeScaleOutPending = errors.New("runtime scale-out pending")
|
||||
var (
|
||||
errRuntimeScaleOutPending = errors.New("runtime scale-out pending")
|
||||
errRuntimeGatewayStartPending = errors.New("runtime gateway start pending")
|
||||
)
|
||||
|
||||
const runtimeRolloutStaleWindowMultiplier = 3
|
||||
const (
|
||||
runtimeRolloutStaleWindowMultiplier = 3
|
||||
runtimeGatewayStartInFlightLimit = 1
|
||||
runtimeSchedulerBatchLimit = 1000
|
||||
)
|
||||
|
||||
type RuntimeGatewayEnvBuilder func(*models.Instance) (map[string]string, error)
|
||||
|
||||
@@ -108,21 +119,22 @@ func NewRuntimeScheduler(
|
||||
tick = 2 * time.Second
|
||||
}
|
||||
s := &RuntimeScheduler{
|
||||
instanceRepo: instanceRepo,
|
||||
podRepo: podRepo,
|
||||
bindingRepo: bindingRepo,
|
||||
rolloutRepo: rolloutRepo,
|
||||
agentClient: agentClient,
|
||||
events: events,
|
||||
leader: leader,
|
||||
deployments: deployments,
|
||||
tick: tick,
|
||||
workspaceRoot: "/workspaces",
|
||||
runtimeNamespace: "clawmanager-system",
|
||||
gatewayPortStart: RuntimeGatewayPortStart,
|
||||
gatewayPortEnd: RuntimeGatewayPortEnd,
|
||||
heartbeatTimeout: 10 * time.Second,
|
||||
maxGatewaysPerPod: RuntimePodCapacity,
|
||||
instanceRepo: instanceRepo,
|
||||
podRepo: podRepo,
|
||||
bindingRepo: bindingRepo,
|
||||
rolloutRepo: rolloutRepo,
|
||||
agentClient: agentClient,
|
||||
events: events,
|
||||
leader: leader,
|
||||
deployments: deployments,
|
||||
tick: tick,
|
||||
workspaceRoot: "/workspaces",
|
||||
runtimeNamespace: "clawmanager-system",
|
||||
gatewayPortStart: RuntimeGatewayPortStart,
|
||||
gatewayPortEnd: RuntimeGatewayPortEnd,
|
||||
heartbeatTimeout: 10 * time.Second,
|
||||
maxGatewaysPerPod: RuntimePodCapacity,
|
||||
gatewayCreateLocks: map[int64]*sync.Mutex{},
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(s)
|
||||
@@ -493,7 +505,7 @@ func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
|
||||
if err := s.reconcileRollouts(ctx); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
creating, err := s.instanceRepo.GetV2Creating(ctx, 100)
|
||||
creating, err := s.instanceRepo.GetV2Creating(ctx, runtimeSchedulerBatchLimit)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
} else {
|
||||
@@ -513,7 +525,7 @@ func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
if err := s.assignInstance(ctx, instance); err != nil {
|
||||
if errors.Is(err, errRuntimeScaleOutPending) {
|
||||
if errors.Is(err, errRuntimeScaleOutPending) || errors.Is(err, errRuntimeGatewayStartPending) {
|
||||
continue
|
||||
}
|
||||
errs = append(errs, fmt.Errorf("assign creating instance %d: %w", instance.ID, err))
|
||||
@@ -522,10 +534,13 @@ func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
desired, err := s.instanceRepo.GetV2DesiredRunning(ctx, 100)
|
||||
desired, err := s.instanceRepo.GetV2DesiredRunning(ctx, runtimeSchedulerBatchLimit)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
} else {
|
||||
if err := s.scaleOutForPendingBacklog(ctx, desired); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
for _, instance := range desired {
|
||||
if !isSchedulerManagedV2Instance(instance) {
|
||||
continue
|
||||
@@ -556,7 +571,7 @@ func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
if assignErr := s.assignInstance(ctx, instance); assignErr != nil {
|
||||
if errors.Is(assignErr, errRuntimeScaleOutPending) {
|
||||
if errors.Is(assignErr, errRuntimeScaleOutPending) || errors.Is(assignErr, errRuntimeGatewayStartPending) {
|
||||
continue
|
||||
}
|
||||
errs = append(errs, fmt.Errorf("assign desired instance %d: %w", instance.ID, assignErr))
|
||||
@@ -619,6 +634,152 @@ func (s *RuntimeScheduler) failoverStalePods(ctx context.Context) error {
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (s *RuntimeScheduler) lockGatewayCreateForPod(podID int64) func() {
|
||||
if s == nil {
|
||||
return func() {}
|
||||
}
|
||||
s.gatewayCreateLocksMu.Lock()
|
||||
if s.gatewayCreateLocks == nil {
|
||||
s.gatewayCreateLocks = map[int64]*sync.Mutex{}
|
||||
}
|
||||
lock := s.gatewayCreateLocks[podID]
|
||||
if lock == nil {
|
||||
lock = &sync.Mutex{}
|
||||
s.gatewayCreateLocks[podID] = lock
|
||||
}
|
||||
s.gatewayCreateLocksMu.Unlock()
|
||||
|
||||
lock.Lock()
|
||||
return lock.Unlock
|
||||
}
|
||||
|
||||
func (s *RuntimeScheduler) podCanStartGateway(ctx context.Context, podID int64) (bool, error) {
|
||||
if s == nil || s.bindingRepo == nil || runtimeGatewayStartInFlightLimit <= 0 {
|
||||
return true, nil
|
||||
}
|
||||
bindings, err := s.bindingRepo.ListByRuntimePodID(ctx, podID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("list runtime pod %d bindings: %w", podID, err)
|
||||
}
|
||||
starting := 0
|
||||
for _, binding := range bindings {
|
||||
switch strings.ToLower(strings.TrimSpace(binding.State)) {
|
||||
case "creating", "starting":
|
||||
starting++
|
||||
}
|
||||
}
|
||||
return starting < runtimeGatewayStartInFlightLimit, nil
|
||||
}
|
||||
|
||||
func (s *RuntimeScheduler) scaleOutForPendingBacklog(ctx context.Context, instances []models.Instance) error {
|
||||
if s == nil || s.bindingRepo == nil || s.podRepo == nil || s.deployments == nil || len(instances) == 0 {
|
||||
return nil
|
||||
}
|
||||
backlogByType := map[string]int{}
|
||||
for _, instance := range instances {
|
||||
if !isSchedulerManagedV2Instance(instance) {
|
||||
continue
|
||||
}
|
||||
if isRuntimeErrorInstance(instance) && !isRecoverableRuntimeSchedulingError(instance) {
|
||||
continue
|
||||
}
|
||||
runtimeType, ok := schedulerRuntimeType(instance)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get binding for backlog instance %d: %w", instance.ID, err)
|
||||
}
|
||||
if binding == nil {
|
||||
backlogByType[runtimeType]++
|
||||
}
|
||||
}
|
||||
for runtimeType, backlog := range backlogByType {
|
||||
if backlog <= 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := s.scaleOutIfBacklogExceedsCapacity(ctx, runtimeType, backlog); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *RuntimeScheduler) scaleOutIfBacklogExceedsCapacity(ctx context.Context, runtimeType string, backlog int) (bool, error) {
|
||||
if s == nil || s.deployments == nil || s.podRepo == nil || backlog <= 0 {
|
||||
return false, nil
|
||||
}
|
||||
pods, err := s.podRepo.List(ctx, runtimeType)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("list %s runtime pods for backlog scale-out: %w", runtimeType, err)
|
||||
}
|
||||
groups := map[string]*runtimeDeploymentCapacity{}
|
||||
for _, pod := range pods {
|
||||
if pod.RuntimeType != runtimeType || pod.State != "ready" || pod.Draining || pod.Capacity <= 0 {
|
||||
continue
|
||||
}
|
||||
namespace := strings.TrimSpace(pod.Namespace)
|
||||
name := strings.TrimSpace(pod.DeploymentName)
|
||||
if namespace == "" || name == "" {
|
||||
continue
|
||||
}
|
||||
key := namespace + "/" + name
|
||||
group := groups[key]
|
||||
if group == nil {
|
||||
group = &runtimeDeploymentCapacity{namespace: namespace, name: name}
|
||||
groups[key] = group
|
||||
}
|
||||
group.active++
|
||||
group.capacityTotal += pod.Capacity
|
||||
group.usedSlots += minInt(pod.UsedSlots, pod.Capacity)
|
||||
}
|
||||
var target *runtimeDeploymentCapacity
|
||||
for _, group := range groups {
|
||||
if group.active == 0 || group.capacityTotal <= 0 {
|
||||
continue
|
||||
}
|
||||
if target == nil || group.active > target.active {
|
||||
target = group
|
||||
}
|
||||
}
|
||||
if target == nil {
|
||||
return false, nil
|
||||
}
|
||||
desiredSlots := target.usedSlots + backlog
|
||||
if desiredSlots <= target.capacityTotal {
|
||||
return false, nil
|
||||
}
|
||||
capacityPerPod := target.capacityTotal / target.active
|
||||
if capacityPerPod <= 0 {
|
||||
capacityPerPod = s.maxGatewaysPerPod
|
||||
}
|
||||
if capacityPerPod <= 0 {
|
||||
capacityPerPod = RuntimePodCapacity
|
||||
}
|
||||
targetReplicas := (desiredSlots + capacityPerPod - 1) / capacityPerPod
|
||||
if targetReplicas <= target.active {
|
||||
targetReplicas = target.active + 1
|
||||
}
|
||||
// Scale one replica at a time so a large batch cannot stampede the node.
|
||||
replicas := int32(minInt(targetReplicas, target.active+1))
|
||||
if err := s.deployments.Scale(ctx, target.namespace, target.name, replicas); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if s.events != nil {
|
||||
if err := s.events.Publish(ctx, "runtime.pool.scaleout", map[string]any{
|
||||
"runtime_type": runtimeType,
|
||||
"namespace": target.namespace,
|
||||
"deployment": target.name,
|
||||
"replicas": replicas,
|
||||
"reason": "pending_backlog",
|
||||
"backlog": backlog,
|
||||
}); err != nil {
|
||||
log.Printf("runtime scheduler publish backlog scale-out event failed: %v", err)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error {
|
||||
if s == nil || s.podRepo == nil {
|
||||
return fmt.Errorf("runtime scheduler pod repository is not configured")
|
||||
@@ -657,15 +818,31 @@ func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.I
|
||||
if pod.AgentEndpoint == nil || strings.TrimSpace(*pod.AgentEndpoint) == "" {
|
||||
continue
|
||||
}
|
||||
unlock := s.lockGatewayCreateForPod(pod.ID)
|
||||
canStart, err := s.podCanStartGateway(ctx, pod.ID)
|
||||
if err != nil {
|
||||
unlock()
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if !canStart {
|
||||
unlock()
|
||||
lastErr = fmt.Errorf("runtime pod %d has gateway start in flight: %w", pod.ID, errRuntimeGatewayStartPending)
|
||||
continue
|
||||
}
|
||||
claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID)
|
||||
if err != nil {
|
||||
unlock()
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if !claimed {
|
||||
unlock()
|
||||
continue
|
||||
}
|
||||
if err := s.createGatewayOnPod(ctx, instance, runtimeType, pod); err != nil {
|
||||
err = s.createGatewayOnPod(ctx, instance, runtimeType, pod)
|
||||
unlock()
|
||||
if err != nil {
|
||||
if releaseErr := s.podRepo.ReleaseSlot(ctx, pod.ID); releaseErr != nil {
|
||||
return errors.Join(err, releaseErr)
|
||||
}
|
||||
@@ -681,10 +858,12 @@ func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.I
|
||||
}
|
||||
|
||||
type runtimeDeploymentCapacity struct {
|
||||
namespace string
|
||||
name string
|
||||
active int
|
||||
full int
|
||||
namespace string
|
||||
name string
|
||||
active int
|
||||
full int
|
||||
capacityTotal int
|
||||
usedSlots int
|
||||
}
|
||||
|
||||
func (s *RuntimeScheduler) scaleOutIfAtCapacity(ctx context.Context, runtimeType string) (bool, error) {
|
||||
@@ -781,7 +960,7 @@ func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance mode
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
gatewayState := normalizeRuntimeGatewayCreateState(resp.Status)
|
||||
lifecycle := NormalizeRuntimeGatewayLifecycle(resp.Status, nil)
|
||||
binding := &models.InstanceRuntimeBinding{
|
||||
InstanceID: instance.ID,
|
||||
RuntimePodID: pod.ID,
|
||||
@@ -790,41 +969,39 @@ func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance mode
|
||||
GatewayPort: resp.Port,
|
||||
GatewayPID: resp.PID,
|
||||
WorkspacePath: workspacePath,
|
||||
State: gatewayState,
|
||||
State: lifecycle.BindingState,
|
||||
Generation: instance.RuntimeGeneration,
|
||||
}
|
||||
if gatewayState == "running" {
|
||||
if lifecycle.Running {
|
||||
binding.LastHealthAt = &now
|
||||
}
|
||||
if err := s.bindingRepo.Create(ctx, binding); err != nil {
|
||||
if err := s.createRuntimeBindingWithStalePortRecovery(ctx, binding); err != nil {
|
||||
return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, false, err)
|
||||
}
|
||||
if !lifecycle.CreateAccepted() {
|
||||
cause := fmt.Errorf("runtime gateway %s returned status %q", resp.GatewayID, strings.TrimSpace(resp.Status))
|
||||
message := cause.Error()
|
||||
var errs []error
|
||||
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &message); err != nil {
|
||||
errs = append(errs, fmt.Errorf("mark instance %d error: %w", instance.ID, err))
|
||||
}
|
||||
errs = append(errs, s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, cause))
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
|
||||
return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err)
|
||||
}
|
||||
instanceState := "creating"
|
||||
var stateMessage *string
|
||||
if gatewayState == "running" {
|
||||
instanceState = "running"
|
||||
} else {
|
||||
message := "runtime gateway starting"
|
||||
stateMessage = &message
|
||||
}
|
||||
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, instanceState, instance.RuntimeGeneration, stateMessage); err != nil {
|
||||
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, lifecycle.InstanceState, instance.RuntimeGeneration, lifecycle.Message); err != nil {
|
||||
return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err)
|
||||
}
|
||||
if s.events != nil {
|
||||
eventType := "runtime.instance.starting"
|
||||
if gatewayState == "running" {
|
||||
eventType = "runtime.instance.running"
|
||||
}
|
||||
if err := s.events.Publish(ctx, eventType, map[string]any{
|
||||
if err := s.events.Publish(ctx, lifecycle.EventType, map[string]any{
|
||||
"instance_id": instance.ID,
|
||||
"runtime_type": runtimeType,
|
||||
"runtime_pod_id": pod.ID,
|
||||
"gateway_id": resp.GatewayID,
|
||||
"gateway_port": resp.Port,
|
||||
"gateway_state": gatewayState,
|
||||
"gateway_state": lifecycle.BindingState,
|
||||
"workspace_path": workspacePath,
|
||||
"generation": instance.RuntimeGeneration,
|
||||
}); err != nil {
|
||||
@@ -834,15 +1011,21 @@ func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance mode
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeRuntimeGatewayCreateState(status string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(status)) {
|
||||
case "running", "ready", "healthy":
|
||||
return "running"
|
||||
default:
|
||||
return "starting"
|
||||
func (s *RuntimeScheduler) createRuntimeBindingWithStalePortRecovery(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
|
||||
if err := s.bindingRepo.Create(ctx, binding); err != nil {
|
||||
deleted, deleteErr := s.bindingRepo.DeleteErrorByRuntimePodIDAndGatewayPort(ctx, binding.RuntimePodID, binding.GatewayPort)
|
||||
if deleteErr != nil {
|
||||
return errors.Join(err, deleteErr)
|
||||
}
|
||||
if deleted == 0 {
|
||||
return err
|
||||
}
|
||||
if retryErr := s.bindingRepo.Create(ctx, binding); retryErr != nil {
|
||||
return errors.Join(err, retryErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runtimeGatewayLinuxIDs(instanceID int, environment map[string]string) (int, int) {
|
||||
linuxID := RuntimeLinuxID(instanceID)
|
||||
if !strings.EqualFold(strings.TrimSpace(environment["CLAWMANAGER_TEAM_ENABLED"]), "true") {
|
||||
@@ -930,7 +1113,8 @@ func isRecoverableRuntimeSchedulingError(instance models.Instance) bool {
|
||||
return false
|
||||
}
|
||||
message := strings.TrimSpace(*instance.RuntimeErrorMessage)
|
||||
return message == fmt.Sprintf("no schedulable %s runtime pod", runtimeType)
|
||||
return message == fmt.Sprintf("no schedulable %s runtime pod", runtimeType) ||
|
||||
strings.Contains(message, fmt.Sprintf("no schedulable %s runtime pod:", runtimeType))
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
|
||||
@@ -3,7 +3,10 @@ package services
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -83,7 +86,7 @@ func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) {
|
||||
if req.req.WorkspacePath != "/workspaces/openclaw/user-45/instance-17" {
|
||||
t.Fatalf("workspace path = %q", req.req.WorkspacePath)
|
||||
}
|
||||
if req.req.PortRange.Start != 20000 || req.req.PortRange.End != 20099 {
|
||||
if req.req.PortRange.Start != RuntimeGatewayPortStart || req.req.PortRange.End != RuntimeGatewayPortEnd {
|
||||
t.Fatalf("port range = %+v", req.req.PortRange)
|
||||
}
|
||||
if req.req.UID != RuntimeLinuxID(17) || req.req.GID != RuntimeLinuxID(17) {
|
||||
@@ -118,6 +121,359 @@ func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeSchedulerStoresStartingGatewayBindingAsCreating(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint := "http://agent.runtime"
|
||||
workspacePath := "/workspaces/openclaw/user-45/instance-72"
|
||||
instanceRepo := newFakeRuntimeInstanceRepo()
|
||||
podRepo := &fakeRuntimePodRepo{
|
||||
pods: map[int64]*models.RuntimePod{
|
||||
9: {
|
||||
ID: 9,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
AgentEndpoint: &endpoint,
|
||||
State: "ready",
|
||||
Capacity: 100,
|
||||
},
|
||||
},
|
||||
schedulable: []models.RuntimePod{{
|
||||
ID: 9,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
AgentEndpoint: &endpoint,
|
||||
State: "ready",
|
||||
Capacity: 100,
|
||||
}},
|
||||
}
|
||||
bindingRepo := newFakeRuntimeBindingRepo()
|
||||
agent := &fakeRuntimeAgentClient{
|
||||
createResponse: &RuntimeAgentCreateGatewayResponse{
|
||||
GatewayID: "gw-72",
|
||||
Port: 20072,
|
||||
Status: "starting",
|
||||
},
|
||||
}
|
||||
scheduler := NewRuntimeScheduler(
|
||||
instanceRepo,
|
||||
podRepo,
|
||||
bindingRepo,
|
||||
&fakeRuntimeRolloutRepo{},
|
||||
agent,
|
||||
&fakeRuntimeEventService{},
|
||||
nil,
|
||||
&fakeRuntimeDeploymentService{},
|
||||
time.Second,
|
||||
)
|
||||
|
||||
err := scheduler.assignInstance(ctx, models.Instance{
|
||||
ID: 72,
|
||||
UserID: 45,
|
||||
Type: RuntimeTypeOpenClaw,
|
||||
RuntimeType: RuntimeBackendGateway,
|
||||
InstanceMode: InstanceModeLite,
|
||||
Status: "creating",
|
||||
WorkspacePath: &workspacePath,
|
||||
RuntimeGeneration: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("assignInstance returned error: %v", err)
|
||||
}
|
||||
|
||||
binding := bindingRepo.bindings[72]
|
||||
if binding == nil {
|
||||
t.Fatal("binding was not created")
|
||||
}
|
||||
if binding.State != "creating" {
|
||||
t.Fatalf("binding state = %q, want creating", binding.State)
|
||||
}
|
||||
if binding.LastHealthAt != nil {
|
||||
t.Fatalf("last health at = %v, want nil until gateway is running", binding.LastHealthAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeSchedulerMarksInstanceErrorWhenGatewayCreateReturnsErrorStatus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint := "http://agent.runtime"
|
||||
workspacePath := "/workspaces/openclaw/user-45/instance-73"
|
||||
instanceRepo := newFakeRuntimeInstanceRepo()
|
||||
podRepo := &fakeRuntimePodRepo{
|
||||
pods: map[int64]*models.RuntimePod{
|
||||
9: {
|
||||
ID: 9,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
AgentEndpoint: &endpoint,
|
||||
State: "ready",
|
||||
Capacity: 100,
|
||||
},
|
||||
},
|
||||
schedulable: []models.RuntimePod{{
|
||||
ID: 9,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
AgentEndpoint: &endpoint,
|
||||
State: "ready",
|
||||
Capacity: 100,
|
||||
}},
|
||||
}
|
||||
bindingRepo := newFakeRuntimeBindingRepo()
|
||||
agent := &fakeRuntimeAgentClient{
|
||||
createResponse: &RuntimeAgentCreateGatewayResponse{
|
||||
GatewayID: "gw-73",
|
||||
Port: 20073,
|
||||
Status: "error",
|
||||
},
|
||||
}
|
||||
events := &fakeRuntimeEventService{}
|
||||
scheduler := NewRuntimeScheduler(
|
||||
instanceRepo,
|
||||
podRepo,
|
||||
bindingRepo,
|
||||
&fakeRuntimeRolloutRepo{},
|
||||
agent,
|
||||
events,
|
||||
nil,
|
||||
&fakeRuntimeDeploymentService{},
|
||||
time.Second,
|
||||
)
|
||||
|
||||
err := scheduler.assignInstance(ctx, models.Instance{
|
||||
ID: 73,
|
||||
UserID: 45,
|
||||
Type: RuntimeTypeOpenClaw,
|
||||
RuntimeType: RuntimeBackendGateway,
|
||||
InstanceMode: InstanceModeLite,
|
||||
Status: "creating",
|
||||
WorkspacePath: &workspacePath,
|
||||
RuntimeGeneration: 2,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("assignInstance returned nil error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `runtime gateway gw-73 returned status "error"`) {
|
||||
t.Fatalf("assignInstance error = %v, want gateway status error", err)
|
||||
}
|
||||
state := instanceRepo.runtimeStates[73]
|
||||
if state.status != "error" || state.generation != 2 || state.message == nil || !strings.Contains(*state.message, `runtime gateway gw-73 returned status "error"`) {
|
||||
t.Fatalf("runtime state = %+v, want error with gateway status message", state)
|
||||
}
|
||||
if _, ok := instanceRepo.workspacePaths[73]; ok {
|
||||
t.Fatalf("workspace path was set after gateway error: %q", instanceRepo.workspacePaths[73])
|
||||
}
|
||||
if bindingRepo.bindings[73] != nil {
|
||||
t.Fatalf("binding remains after gateway error: %+v", bindingRepo.bindings[73])
|
||||
}
|
||||
if bindingRepo.deleteCalls[73] != 1 {
|
||||
t.Fatalf("binding delete calls = %d, want 1", bindingRepo.deleteCalls[73])
|
||||
}
|
||||
if got := len(agent.deleteRequests); got != 1 {
|
||||
t.Fatalf("DeleteGateway calls = %d, want 1", got)
|
||||
}
|
||||
if agent.deleteRequests[0].gatewayID != "gw-73" {
|
||||
t.Fatalf("deleted gateway = %q, want gw-73", agent.deleteRequests[0].gatewayID)
|
||||
}
|
||||
if podRepo.releases[9] != 1 {
|
||||
t.Fatalf("pod releases = %d, want 1", podRepo.releases[9])
|
||||
}
|
||||
if got := len(events.published); got != 0 {
|
||||
t.Fatalf("published events = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
func TestRuntimeSchedulerThrottlesGatewayStartsWhenPodHasCreatingBinding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint := "http://agent.runtime"
|
||||
workspacePath := "/workspaces/openclaw/user-45/instance-83"
|
||||
pod := models.RuntimePod{
|
||||
ID: 9,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
AgentEndpoint: &endpoint,
|
||||
State: "ready",
|
||||
Capacity: 100,
|
||||
}
|
||||
podRepo := &fakeRuntimePodRepo{
|
||||
pods: map[int64]*models.RuntimePod{9: &pod},
|
||||
schedulable: []models.RuntimePod{pod},
|
||||
}
|
||||
bindingRepo := newFakeRuntimeBindingRepo()
|
||||
bindingRepo.bindings[82] = &models.InstanceRuntimeBinding{
|
||||
InstanceID: 82,
|
||||
RuntimePodID: 9,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
State: "creating",
|
||||
Generation: 1,
|
||||
}
|
||||
agent := &fakeRuntimeAgentClient{
|
||||
createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-83", Port: 20083, Status: "running"},
|
||||
}
|
||||
scheduler := NewRuntimeScheduler(
|
||||
newFakeRuntimeInstanceRepo(),
|
||||
podRepo,
|
||||
bindingRepo,
|
||||
&fakeRuntimeRolloutRepo{},
|
||||
agent,
|
||||
&fakeRuntimeEventService{},
|
||||
nil,
|
||||
&fakeRuntimeDeploymentService{},
|
||||
time.Second,
|
||||
)
|
||||
|
||||
err := scheduler.assignInstance(ctx, models.Instance{
|
||||
ID: 83,
|
||||
UserID: 45,
|
||||
Type: RuntimeTypeOpenClaw,
|
||||
RuntimeType: RuntimeBackendGateway,
|
||||
InstanceMode: InstanceModeLite,
|
||||
Status: "creating",
|
||||
WorkspacePath: &workspacePath,
|
||||
RuntimeGeneration: 1,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("assignInstance returned nil error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "gateway start in flight") {
|
||||
t.Fatalf("assignInstance error = %v, want gateway start in flight", err)
|
||||
}
|
||||
if got := len(agent.createRequests); got != 0 {
|
||||
t.Fatalf("CreateGateway calls = %d, want 0", got)
|
||||
}
|
||||
if podRepo.claims[9] != 0 {
|
||||
t.Fatalf("pod claims = %d, want 0", podRepo.claims[9])
|
||||
}
|
||||
}
|
||||
func TestRuntimeSchedulerReconcileWaitsWhenGatewayStartInFlight(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint := "http://agent.runtime"
|
||||
workspacePath := "/workspaces/openclaw/user-45/instance-84"
|
||||
instanceRepo := newFakeRuntimeInstanceRepo()
|
||||
instanceRepo.creating = []models.Instance{{
|
||||
ID: 84,
|
||||
UserID: 45,
|
||||
Type: RuntimeTypeOpenClaw,
|
||||
RuntimeType: RuntimeBackendGateway,
|
||||
InstanceMode: InstanceModeLite,
|
||||
Status: "creating",
|
||||
WorkspacePath: &workspacePath,
|
||||
RuntimeGeneration: 1,
|
||||
}}
|
||||
pod := models.RuntimePod{
|
||||
ID: 9,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
AgentEndpoint: &endpoint,
|
||||
State: "ready",
|
||||
Capacity: 100,
|
||||
}
|
||||
podRepo := &fakeRuntimePodRepo{
|
||||
pods: map[int64]*models.RuntimePod{9: &pod},
|
||||
schedulable: []models.RuntimePod{pod},
|
||||
}
|
||||
bindingRepo := newFakeRuntimeBindingRepo()
|
||||
bindingRepo.bindings[82] = &models.InstanceRuntimeBinding{
|
||||
InstanceID: 82,
|
||||
RuntimePodID: 9,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
State: "creating",
|
||||
Generation: 1,
|
||||
}
|
||||
agent := &fakeRuntimeAgentClient{
|
||||
createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-84", Port: 20084, Status: "running"},
|
||||
}
|
||||
scheduler := NewRuntimeScheduler(
|
||||
instanceRepo,
|
||||
podRepo,
|
||||
bindingRepo,
|
||||
&fakeRuntimeRolloutRepo{},
|
||||
agent,
|
||||
&fakeRuntimeEventService{},
|
||||
nil,
|
||||
&fakeRuntimeDeploymentService{},
|
||||
time.Second,
|
||||
)
|
||||
|
||||
if err := scheduler.reconcile(ctx); err != nil {
|
||||
t.Fatalf("reconcile returned error: %v", err)
|
||||
}
|
||||
if got := len(agent.createRequests); got != 0 {
|
||||
t.Fatalf("CreateGateway calls = %d, want 0", got)
|
||||
}
|
||||
if podRepo.claims[9] != 0 {
|
||||
t.Fatalf("pod claims = %d, want 0", podRepo.claims[9])
|
||||
}
|
||||
if state, ok := instanceRepo.runtimeStates[84]; ok {
|
||||
t.Fatalf("runtime state was updated to %+v, want waiting without error", state)
|
||||
}
|
||||
}
|
||||
func TestRuntimeSchedulerSerializesGatewayCreationPerPod(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint := "http://agent.runtime"
|
||||
workspace81 := "/workspaces/openclaw/user-45/instance-81"
|
||||
workspace82 := "/workspaces/openclaw/user-45/instance-82"
|
||||
instances := []models.Instance{
|
||||
{
|
||||
ID: 81,
|
||||
UserID: 45,
|
||||
Type: RuntimeTypeOpenClaw,
|
||||
RuntimeType: RuntimeBackendGateway,
|
||||
InstanceMode: InstanceModeLite,
|
||||
Status: "creating",
|
||||
WorkspacePath: &workspace81,
|
||||
RuntimeGeneration: 1,
|
||||
},
|
||||
{
|
||||
ID: 82,
|
||||
UserID: 45,
|
||||
Type: RuntimeTypeOpenClaw,
|
||||
RuntimeType: RuntimeBackendGateway,
|
||||
InstanceMode: InstanceModeLite,
|
||||
Status: "creating",
|
||||
WorkspacePath: &workspace82,
|
||||
RuntimeGeneration: 1,
|
||||
},
|
||||
}
|
||||
pod := models.RuntimePod{
|
||||
ID: 9,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
AgentEndpoint: &endpoint,
|
||||
State: "ready",
|
||||
Capacity: 100,
|
||||
}
|
||||
podRepo := &fakeRuntimePodRepo{
|
||||
pods: map[int64]*models.RuntimePod{9: &pod},
|
||||
schedulable: []models.RuntimePod{pod},
|
||||
}
|
||||
bindingRepo := newFakeRuntimeBindingRepo()
|
||||
agent := &fakeRuntimeAgentClient{
|
||||
createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw", Port: 20081, Status: "running"},
|
||||
createDelay: 25 * time.Millisecond,
|
||||
}
|
||||
scheduler := NewRuntimeScheduler(
|
||||
newFakeRuntimeInstanceRepo(),
|
||||
podRepo,
|
||||
bindingRepo,
|
||||
&fakeRuntimeRolloutRepo{},
|
||||
agent,
|
||||
&fakeRuntimeEventService{},
|
||||
nil,
|
||||
&fakeRuntimeDeploymentService{},
|
||||
time.Second,
|
||||
)
|
||||
|
||||
errs := make(chan error, len(instances))
|
||||
for _, instance := range instances {
|
||||
instance := instance
|
||||
go func() {
|
||||
errs <- scheduler.assignInstance(ctx, instance)
|
||||
}()
|
||||
}
|
||||
for range instances {
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatalf("assignInstance returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt32(&agent.maxActiveCreates); got != 1 {
|
||||
t.Fatalf("concurrent CreateGateway calls = %d, want 1", got)
|
||||
}
|
||||
if got := len(agent.createRequests); got != 2 {
|
||||
t.Fatalf("CreateGateway calls = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
func TestRuntimeSchedulerPassesGatewayEnvironmentToRuntimeAgent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint := "http://agent.runtime"
|
||||
@@ -371,6 +727,62 @@ func TestRuntimeSchedulerScalesOutWhenAllReadyPodsAtCapacity(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeSchedulerScalesOutWhenBacklogExceedsReadyCapacity(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint1 := "http://pod-1.runtime"
|
||||
endpoint2 := "http://pod-2.runtime"
|
||||
podRepo := &fakeRuntimePodRepo{
|
||||
pods: map[int64]*models.RuntimePod{
|
||||
5: {
|
||||
ID: 5,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
Namespace: "clawmanager-system",
|
||||
DeploymentName: "openclaw-runtime",
|
||||
AgentEndpoint: &endpoint1,
|
||||
State: "ready",
|
||||
Capacity: 100,
|
||||
UsedSlots: 99,
|
||||
},
|
||||
6: {
|
||||
ID: 6,
|
||||
RuntimeType: RuntimeTypeOpenClaw,
|
||||
Namespace: "clawmanager-system",
|
||||
DeploymentName: "openclaw-runtime",
|
||||
AgentEndpoint: &endpoint2,
|
||||
State: "ready",
|
||||
Capacity: 100,
|
||||
UsedSlots: 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
deployments := &fakeRuntimeDeploymentService{}
|
||||
scheduler := NewRuntimeScheduler(
|
||||
newFakeRuntimeInstanceRepo(),
|
||||
podRepo,
|
||||
newFakeRuntimeBindingRepo(),
|
||||
&fakeRuntimeRolloutRepo{},
|
||||
&fakeRuntimeAgentClient{},
|
||||
NewRuntimeEventService(nil),
|
||||
nil,
|
||||
deployments,
|
||||
time.Second,
|
||||
)
|
||||
|
||||
scaled, err := scheduler.scaleOutIfBacklogExceedsCapacity(ctx, RuntimeTypeOpenClaw, 120)
|
||||
if err != nil {
|
||||
t.Fatalf("scaleOutIfBacklogExceedsCapacity returned error: %v", err)
|
||||
}
|
||||
if !scaled {
|
||||
t.Fatal("scaleOutIfBacklogExceedsCapacity returned scaled=false")
|
||||
}
|
||||
if got := len(deployments.scales); got != 1 {
|
||||
t.Fatalf("deployment Scale calls = %d, want 1", got)
|
||||
}
|
||||
scale := deployments.scales[0]
|
||||
if scale.namespace != "clawmanager-system" || scale.name != "openclaw-runtime" || scale.replicas != 3 {
|
||||
t.Fatalf("deployment Scale call = %+v, want clawmanager-system/openclaw-runtime replicas 3", scale)
|
||||
}
|
||||
}
|
||||
func TestRuntimeSchedulerDoesNotScaleOutWhenGatewayCreateFailsWithFreeSlot(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint := "http://pod-1.runtime"
|
||||
@@ -480,7 +892,7 @@ func TestRuntimeSchedulerReconcileRetriesRecoverableNoSchedulableError(t *testin
|
||||
ctx := context.Background()
|
||||
endpoint := "http://agent.runtime"
|
||||
workspacePath := "/workspaces/openclaw/user-46/instance-68"
|
||||
errorMessage := "no schedulable openclaw runtime pod"
|
||||
errorMessage := "no schedulable openclaw runtime pod: runtime agent conflict: no free port"
|
||||
instanceRepo := newFakeRuntimeInstanceRepo()
|
||||
instanceRepo.desiredRunning = []models.Instance{{
|
||||
ID: 68,
|
||||
@@ -850,6 +1262,101 @@ func TestRuntimeSchedulerCleansUpGatewayAndReleasesSlotWhenBindingCreateFails(t
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeSchedulerClearsStaleErrorBindingAndRetriesBindingCreate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint := "http://agent.runtime"
|
||||
instanceRepo := newFakeRuntimeInstanceRepo()
|
||||
podRepo := &fakeRuntimePodRepo{
|
||||
pods: map[int64]*models.RuntimePod{
|
||||
9: {ID: 9, RuntimeType: RuntimeTypeHermes, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
|
||||
},
|
||||
schedulable: []models.RuntimePod{
|
||||
{ID: 9, RuntimeType: RuntimeTypeHermes, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
|
||||
},
|
||||
}
|
||||
bindingRepo := newFakeRuntimeBindingRepo()
|
||||
bindingRepo.enforceUniqueGateway = true
|
||||
bindingRepo.bindings[141] = &models.InstanceRuntimeBinding{
|
||||
InstanceID: 141,
|
||||
RuntimePodID: 9,
|
||||
RuntimeType: RuntimeTypeHermes,
|
||||
GatewayID: "gw-141-15",
|
||||
GatewayPort: 20000,
|
||||
State: "error",
|
||||
Generation: 15,
|
||||
}
|
||||
agent := &fakeRuntimeAgentClient{
|
||||
createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-218-5", Port: 20000, Status: "running"},
|
||||
}
|
||||
scheduler := NewRuntimeScheduler(
|
||||
instanceRepo,
|
||||
podRepo,
|
||||
bindingRepo,
|
||||
&fakeRuntimeRolloutRepo{},
|
||||
agent,
|
||||
NewRuntimeEventService(nil),
|
||||
nil,
|
||||
&fakeRuntimeDeploymentService{},
|
||||
time.Second,
|
||||
)
|
||||
|
||||
err := scheduler.assignInstance(ctx, models.Instance{
|
||||
ID: 218,
|
||||
UserID: 1,
|
||||
Type: RuntimeTypeHermes,
|
||||
RuntimeType: RuntimeBackendGateway,
|
||||
InstanceMode: InstanceModeLite,
|
||||
WorkspacePath: ptrString("/workspaces/hermes/user-1/instance-218"),
|
||||
MemoryGB: 1,
|
||||
DiskGB: 1,
|
||||
RuntimeGeneration: 5,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("assignInstance returned error: %v", err)
|
||||
}
|
||||
if bindingRepo.bindings[141] != nil {
|
||||
t.Fatal("stale error binding still blocks the gateway port")
|
||||
}
|
||||
binding := bindingRepo.bindings[218]
|
||||
if binding == nil {
|
||||
t.Fatal("new binding was not created after stale cleanup")
|
||||
}
|
||||
if binding.RuntimePodID != 9 || binding.GatewayPort != 20000 || binding.State != "running" {
|
||||
t.Fatalf("new binding = %+v, want pod 9 port 20000 running", binding)
|
||||
}
|
||||
if got := bindingRepo.deleteByPodPortCalls["9/20000"]; got != 1 {
|
||||
t.Fatalf("stale pod/port cleanup calls = %d, want 1", got)
|
||||
}
|
||||
if got := len(agent.deleteRequests); got != 0 {
|
||||
t.Fatalf("DeleteGateway calls = %d, want 0", got)
|
||||
}
|
||||
if podRepo.releases[9] != 0 {
|
||||
t.Fatalf("pod releases = %d, want 0", podRepo.releases[9])
|
||||
}
|
||||
}
|
||||
func TestRuntimeSchedulerDoesNotClearStartingBindingOnGatewayPortConflict(t *testing.T) {
|
||||
bindingRepo := newFakeRuntimeBindingRepo()
|
||||
bindingRepo.bindings[218] = &models.InstanceRuntimeBinding{
|
||||
InstanceID: 218,
|
||||
RuntimePodID: 9,
|
||||
RuntimeType: RuntimeTypeHermes,
|
||||
GatewayID: "gw-218-7",
|
||||
GatewayPort: 20000,
|
||||
State: "starting",
|
||||
Generation: 7,
|
||||
}
|
||||
|
||||
deleted, err := bindingRepo.DeleteErrorByRuntimePodIDAndGatewayPort(context.Background(), 9, 20000)
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteErrorByRuntimePodIDAndGatewayPort returned error: %v", err)
|
||||
}
|
||||
if deleted != 0 {
|
||||
t.Fatalf("deleted bindings = %d, want 0", deleted)
|
||||
}
|
||||
if bindingRepo.bindings[218] == nil || bindingRepo.bindings[218].State != "starting" {
|
||||
t.Fatalf("starting binding was removed or changed: %+v", bindingRepo.bindings[218])
|
||||
}
|
||||
}
|
||||
func TestRuntimeSchedulerCleansUpGatewayBindingAndSlotWhenWorkspacePathUpdateFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
endpoint := "http://agent.runtime"
|
||||
@@ -1563,6 +2070,7 @@ func (r *fakeRuntimeInstanceRepo) Update(instance *models.Instance) error { retu
|
||||
func (r *fakeRuntimeInstanceRepo) Delete(id int) error { return nil }
|
||||
|
||||
type fakeRuntimePodRepo struct {
|
||||
mu sync.Mutex
|
||||
pods map[int64]*models.RuntimePod
|
||||
schedulable []models.RuntimePod
|
||||
claims map[int64]int
|
||||
@@ -1602,6 +2110,8 @@ func (r *fakeRuntimePodRepo) ListSchedulable(ctx context.Context, runtimeType st
|
||||
return r.schedulable, nil
|
||||
}
|
||||
func (r *fakeRuntimePodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.claims == nil {
|
||||
r.claims = map[int64]int{}
|
||||
}
|
||||
@@ -1643,12 +2153,15 @@ func (r *fakeRuntimePodRepo) UpdateMetrics(ctx context.Context, podID int64, met
|
||||
}
|
||||
|
||||
type fakeRuntimeBindingRepo struct {
|
||||
mu sync.Mutex
|
||||
bindings map[int]*models.InstanceRuntimeBinding
|
||||
deleteCalls map[int]int
|
||||
deleteAndReleaseCalls map[int]int
|
||||
runningErr error
|
||||
getErr error
|
||||
createErr error
|
||||
enforceUniqueGateway bool
|
||||
deleteByPodPortCalls map[string]int
|
||||
}
|
||||
|
||||
func newFakeRuntimeBindingRepo() *fakeRuntimeBindingRepo {
|
||||
@@ -1656,24 +2169,38 @@ func newFakeRuntimeBindingRepo() *fakeRuntimeBindingRepo {
|
||||
bindings: map[int]*models.InstanceRuntimeBinding{},
|
||||
deleteCalls: map[int]int{},
|
||||
deleteAndReleaseCalls: map[int]int{},
|
||||
deleteByPodPortCalls: map[string]int{},
|
||||
}
|
||||
}
|
||||
|
||||
func (r *fakeRuntimeBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.createErr != nil {
|
||||
return r.createErr
|
||||
}
|
||||
if r.enforceUniqueGateway {
|
||||
for _, existing := range r.bindings {
|
||||
if existing != nil && existing.InstanceID != binding.InstanceID && existing.RuntimePodID == binding.RuntimePodID && existing.GatewayPort == binding.GatewayPort {
|
||||
return fmt.Errorf("failed to create instance runtime binding: duplicate gateway")
|
||||
}
|
||||
}
|
||||
}
|
||||
copy := *binding
|
||||
r.bindings[binding.InstanceID] = ©
|
||||
return nil
|
||||
}
|
||||
func (r *fakeRuntimeBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.getErr != nil {
|
||||
return nil, r.getErr
|
||||
}
|
||||
return r.bindings[instanceID], nil
|
||||
}
|
||||
func (r *fakeRuntimeBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.runningErr != nil {
|
||||
return nil, r.runningErr
|
||||
}
|
||||
@@ -1684,6 +2211,8 @@ func (r *fakeRuntimeBindingRepo) GetRunningByInstanceID(ctx context.Context, ins
|
||||
return binding, nil
|
||||
}
|
||||
func (r *fakeRuntimeBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var bindings []models.InstanceRuntimeBinding
|
||||
for _, binding := range r.bindings {
|
||||
if binding.RuntimePodID == runtimePodID {
|
||||
@@ -1701,6 +2230,21 @@ func (r *fakeRuntimeBindingRepo) UpdateRunning(ctx context.Context, instanceID i
|
||||
func (r *fakeRuntimeBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
|
||||
return nil
|
||||
}
|
||||
func (r *fakeRuntimeBindingRepo) DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
key := fmt.Sprintf("%d/%d", runtimePodID, gatewayPort)
|
||||
r.deleteByPodPortCalls[key]++
|
||||
var deleted int64
|
||||
for instanceID, binding := range r.bindings {
|
||||
if binding == nil || binding.RuntimePodID != runtimePodID || binding.GatewayPort != gatewayPort || binding.State != "error" {
|
||||
continue
|
||||
}
|
||||
delete(r.bindings, instanceID)
|
||||
deleted++
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
func (r *fakeRuntimeBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
|
||||
r.deleteCalls[instanceID]++
|
||||
delete(r.bindings, instanceID)
|
||||
@@ -1711,6 +2255,15 @@ func (r *fakeRuntimeBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Co
|
||||
delete(r.bindings, instanceID)
|
||||
return nil
|
||||
}
|
||||
func (r *fakeRuntimeBindingRepo) DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error) {
|
||||
binding := r.bindings[instanceID]
|
||||
if binding == nil || binding.RuntimePodID != runtimePodID || binding.Generation != generation || binding.State != "running" {
|
||||
return false, nil
|
||||
}
|
||||
r.deleteAndReleaseCalls[instanceID]++
|
||||
delete(r.bindings, instanceID)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type fakeRuntimeRolloutRepo struct {
|
||||
rollouts map[int64]*models.RuntimeRollout
|
||||
@@ -1755,12 +2308,17 @@ func (r *fakeRuntimeRolloutRepo) UpdateStatus(ctx context.Context, id int64, sta
|
||||
}
|
||||
|
||||
type fakeRuntimeAgentClient struct {
|
||||
mu sync.Mutex
|
||||
createResponse *RuntimeAgentCreateGatewayResponse
|
||||
createErr error
|
||||
deleteErr error
|
||||
createDelay time.Duration
|
||||
createRequests []fakeCreateGatewayRequest
|
||||
deleteRequests []fakeDeleteGatewayRequest
|
||||
drainEndpoints []string
|
||||
|
||||
activeCreates int32
|
||||
maxActiveCreates int32
|
||||
}
|
||||
|
||||
type fakeCreateGatewayRequest struct {
|
||||
@@ -1775,6 +2333,23 @@ type fakeDeleteGatewayRequest struct {
|
||||
|
||||
func (c *fakeRuntimeAgentClient) Health(ctx context.Context, endpoint string) error { return nil }
|
||||
func (c *fakeRuntimeAgentClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) {
|
||||
active := atomic.AddInt32(&c.activeCreates, 1)
|
||||
defer atomic.AddInt32(&c.activeCreates, -1)
|
||||
for {
|
||||
current := atomic.LoadInt32(&c.maxActiveCreates)
|
||||
if active <= current || atomic.CompareAndSwapInt32(&c.maxActiveCreates, current, active) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if c.createDelay > 0 {
|
||||
select {
|
||||
case <-time.After(c.createDelay):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.createRequests = append(c.createRequests, fakeCreateGatewayRequest{endpoint: endpoint, req: req})
|
||||
if c.createErr != nil {
|
||||
return nil, c.createErr
|
||||
|
||||
@@ -6193,6 +6193,8 @@ spec:
|
||||
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
|
||||
- name: TEAM_REDIS_URL
|
||||
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
|
||||
- name: CLAWMANAGER_DESKTOP_DIRECT_PROXY
|
||||
value: "true"
|
||||
- name: CLAWMANAGER_TEAM_REDIS_URL
|
||||
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
|
||||
- name: CLAWMANAGER_TEAM_MANAGER_BASE_URL
|
||||
@@ -6208,7 +6210,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_SCHEDULER_ENABLED
|
||||
value: "true"
|
||||
- name: RUNTIME_HEARTBEAT_TIMEOUT
|
||||
@@ -6305,7 +6307,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_AGENT_LISTEN_ADDR
|
||||
value: "0.0.0.0:19090"
|
||||
- name: RUNTIME_AGENT_PUBLIC_PORT
|
||||
@@ -6392,7 +6394,7 @@ spec:
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: HERMES_TUI_DIR
|
||||
value: /usr/local/lib/hermes-agent/ui-tui
|
||||
- name: CLAWMANAGER_AGENT_CONTROL_TOKEN
|
||||
@@ -6414,7 +6416,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_AGENT_CONTROL_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -1368,7 +1368,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_SCHEDULER_ENABLED
|
||||
value: "true"
|
||||
- name: RUNTIME_HEARTBEAT_TIMEOUT
|
||||
@@ -1465,7 +1465,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_AGENT_LISTEN_ADDR
|
||||
value: "0.0.0.0:19090"
|
||||
- name: RUNTIME_AGENT_PUBLIC_PORT
|
||||
@@ -1552,7 +1552,7 @@ spec:
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: HERMES_TUI_DIR
|
||||
value: /usr/local/lib/hermes-agent/ui-tui
|
||||
- name: CLAWMANAGER_AGENT_CONTROL_TOKEN
|
||||
@@ -1574,7 +1574,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_AGENT_CONTROL_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -6193,6 +6193,8 @@ spec:
|
||||
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
|
||||
- name: TEAM_REDIS_URL
|
||||
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
|
||||
- name: CLAWMANAGER_DESKTOP_DIRECT_PROXY
|
||||
value: "true"
|
||||
- name: CLAWMANAGER_TEAM_REDIS_URL
|
||||
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
|
||||
- name: CLAWMANAGER_TEAM_MANAGER_BASE_URL
|
||||
@@ -6208,7 +6210,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_SCHEDULER_ENABLED
|
||||
value: "true"
|
||||
- name: RUNTIME_HEARTBEAT_TIMEOUT
|
||||
@@ -6305,7 +6307,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_AGENT_LISTEN_ADDR
|
||||
value: "0.0.0.0:19090"
|
||||
- name: RUNTIME_AGENT_PUBLIC_PORT
|
||||
@@ -6392,7 +6394,7 @@ spec:
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: HERMES_TUI_DIR
|
||||
value: /usr/local/lib/hermes-agent/ui-tui
|
||||
- name: CLAWMANAGER_AGENT_CONTROL_TOKEN
|
||||
@@ -6414,7 +6416,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_AGENT_CONTROL_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -1368,7 +1368,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_SCHEDULER_ENABLED
|
||||
value: "true"
|
||||
- name: RUNTIME_HEARTBEAT_TIMEOUT
|
||||
@@ -1465,7 +1465,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_AGENT_LISTEN_ADDR
|
||||
value: "0.0.0.0:19090"
|
||||
- name: RUNTIME_AGENT_PUBLIC_PORT
|
||||
@@ -1552,7 +1552,7 @@ spec:
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: HERMES_TUI_DIR
|
||||
value: /usr/local/lib/hermes-agent/ui-tui
|
||||
- name: CLAWMANAGER_AGENT_CONTROL_TOKEN
|
||||
@@ -1574,7 +1574,7 @@ spec:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_AGENT_CONTROL_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -37,3 +37,89 @@ End users use ClawManager to:
|
||||
2. Review Agent Control Plane if your focus is runtime visibility and operations.
|
||||
3. Review Resource Management and Security Center if you want reusable channels, skills, and scan-backed workflows.
|
||||
4. Create a Team from a role template when a task benefits from coordinated planning, delivery, and review.
|
||||
|
||||
## Desktop Clipboard
|
||||
|
||||
Desktop-mode instances based on WebTop, including OpenClaw Pro and Hermes, use Selkies for browser-to-desktop clipboard synchronization. New instances enable clipboard synchronization in both directions by default.
|
||||
|
||||
| Direction | Default behavior |
|
||||
| --- | --- |
|
||||
| Local browser / host to WebTop | Enabled. Copy content locally, focus an application inside WebTop, then use `Ctrl+V` in that application. |
|
||||
| WebTop to local browser / host | Enabled. Copy content inside WebTop to make it available locally. |
|
||||
|
||||
Copying locally only updates the WebTop clipboard. It does not automatically place content into a chat box, terminal, or editor: first focus the target application inside the remote desktop, then paste there.
|
||||
|
||||
### Change clipboard behavior for one new instance
|
||||
|
||||
In the instance creation page, use **Custom variables**. Values must be entered as plain text: do not include Markdown backticks, quotes, or spaces.
|
||||
|
||||
#### Disable clipboard synchronization in both directions
|
||||
|
||||
Use this when Unicode/IME input occasionally inserts old clipboard content, particularly under a busy desktop or browser workload.
|
||||
|
||||
| Variable | Value |
|
||||
| --- | --- |
|
||||
| `SELKIES_CLIPBOARD_ENABLED` | `false|locked` |
|
||||
| `SELKIES_CLIPBOARD_IN_ENABLED` | `false|locked` |
|
||||
| `SELKIES_CLIPBOARD_OUT_ENABLED` | `false|locked` |
|
||||
|
||||
#### Allow only local browser / host to WebTop
|
||||
|
||||
Use this when users need to paste external content into WebTop but must not copy data from WebTop back to the local machine.
|
||||
|
||||
| Variable | Value |
|
||||
| --- | --- |
|
||||
| `SELKIES_CLIPBOARD_ENABLED` | `true|locked` |
|
||||
| `SELKIES_CLIPBOARD_IN_ENABLED` | `true|locked` |
|
||||
| `SELKIES_CLIPBOARD_OUT_ENABLED` | `false|locked` |
|
||||
|
||||
`SELKIES_CLIPBOARD_ENABLED` is a boolean setting. Do not set it to `in|locked` or `out|locked`: Selkies interprets those as disabled. The `in` and `out` modes are calculated internally from the three boolean settings above.
|
||||
|
||||
For legacy Kasm-compatible images, also set `KASM_SVC_SEND_CUT_TEXT=-SendCutText 0` and `KASM_SVC_ACCEPT_CUT_TEXT=-AcceptCutText 0` when disabling all clipboard synchronization. Current Selkies WebSocket desktops rely on the three `SELKIES_CLIPBOARD_*` settings.
|
||||
|
||||
### When changes take effect
|
||||
|
||||
Clipboard environment variables are read when the desktop Pod starts. They affect only the new instance created with those variables. To change an existing instance, first update its Deployment environment through the control plane or Kubernetes, then restart its Pod; alternatively, create a new instance with the desired custom variables. Changing an environment variable in a terminal inside a running WebTop session has no effect.
|
||||
|
||||
### Verify the effective configuration
|
||||
|
||||
Inside the WebTop terminal, run:
|
||||
|
||||
```bash
|
||||
env | grep SELKIES_CLIPBOARD
|
||||
```
|
||||
|
||||
For bidirectional synchronization, the output must be exactly:
|
||||
|
||||
```text
|
||||
SELKIES_CLIPBOARD_ENABLED=true|locked
|
||||
SELKIES_CLIPBOARD_IN_ENABLED=true|locked
|
||||
SELKIES_CLIPBOARD_OUT_ENABLED=true|locked
|
||||
```
|
||||
|
||||
Runtime logs provide the authoritative confirmation. A healthy inbound-only configuration reports:
|
||||
|
||||
```text
|
||||
clipboard_enabled: (True, True)
|
||||
clipboard_in_enabled: (True, True)
|
||||
clipboard_out_enabled: (False, True)
|
||||
```
|
||||
|
||||
When content from the local browser is copied into WebTop, the runtime logs:
|
||||
|
||||
```text
|
||||
Clipboard direction=inbound ... status=applied
|
||||
```
|
||||
|
||||
If the logs instead show `clipboard_enabled: (False, ...)`, check for an invalid value. A common mistake is entering a leading backtick, such as `` `true`` rather than `true|locked`.
|
||||
|
||||
### IME and Unicode input troubleshooting
|
||||
|
||||
KWin/Wayland can use an internal clipboard fallback to inject Unicode characters. Disabling Selkies clipboard synchronization prevents local clipboard content from being synchronized into WebTop, but it does not remove that internal Unicode fallback. If a user reports that Chinese, Hangul, or another IME input sporadically becomes clipboard text:
|
||||
|
||||
1. Create a diagnostic instance with both clipboard directions disabled.
|
||||
2. Keep a unique sentinel value in the local clipboard and repeatedly enter Unicode text in the affected WebTop application.
|
||||
3. Confirm that the sentinel appears only after an explicit paste, never as part of ordinary typing.
|
||||
4. Review the runtime logs for `Clipboard direction=inbound ... status=applied` while reproducing the issue.
|
||||
|
||||
For a permanent runtime-level solution, use a Unicode input path that does not temporarily write to the system clipboard. Clipboard environment overrides are an instance-level mitigation and should be preferred when the issue is limited to a particular user or workload.
|
||||
|
||||
@@ -71,7 +71,7 @@ agent 至少支持以下环境变量:
|
||||
| `CLAWMANAGER_RUNTIME_TYPE` | runtime 类型,例如 `openclaw`、`hermes`。 |
|
||||
| `RUNTIME_WORKSPACE_ROOT` | workspace 根目录,默认 `/workspaces`。 |
|
||||
| `RUNTIME_GATEWAY_PORT_START` | gateway 端口起始值,默认 `20000`。 |
|
||||
| `RUNTIME_GATEWAY_PORT_END` | gateway 端口结束值,默认 `20099`。 |
|
||||
| `RUNTIME_GATEWAY_PORT_END` | gateway 端口结束值,默认 `20299`。 |
|
||||
| `RUNTIME_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用。 |
|
||||
| `RUNTIME_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用。 |
|
||||
| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址,推荐 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。 |
|
||||
@@ -108,7 +108,7 @@ Content-Type: application/json
|
||||
"workspace_path": "/workspaces/hermes/user-45/instance-123",
|
||||
"port_range": {
|
||||
"start": 20000,
|
||||
"end": 20099
|
||||
"end": 20299
|
||||
},
|
||||
"uid": 200123,
|
||||
"gid": 200123,
|
||||
|
||||
@@ -101,7 +101,7 @@ Request body:
|
||||
"workspace_path": "/workspaces/openclaw/user-45/instance-123",
|
||||
"port_range": {
|
||||
"start": 20000,
|
||||
"end": 20099
|
||||
"end": 20299
|
||||
},
|
||||
"uid": 200123,
|
||||
"gid": 200123,
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
| --- | --- |
|
||||
| workspace root | `/workspaces` |
|
||||
| workspace path | `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` |
|
||||
| gateway 端口范围 | `20000-20099` |
|
||||
| gateway 端口范围 | `20000-20299` |
|
||||
| 单 Runtime Pod 容量 | `100` |
|
||||
| 实例 UID/GID | `200000 + instance_id` |
|
||||
| agent 控制端口 | `19090` |
|
||||
@@ -338,7 +338,7 @@ POST /v1/gateways
|
||||
"workspace_path": "/workspaces/openclaw/user-45/instance-123",
|
||||
"port_range": {
|
||||
"start": 20000,
|
||||
"end": 20099
|
||||
"end": 20299
|
||||
},
|
||||
"uid": 200123,
|
||||
"gid": 200123,
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
| ClawManager 调 agent | `GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{id}`、`POST /v1/drain` | 不适用 |
|
||||
| agent 上报 ClawManager | `/api/v1/runtime-agent/*` | `/api/v1/agent/*` |
|
||||
| 用户实例进程 | 每个 Lite 实例是一个 gateway 子进程 | 整个桌面容器就是实例 |
|
||||
| 用户访问端口 | agent 分配 `20000-20099` 中的 gateway 端口 | Webtop/KasmVNC `3001` |
|
||||
| 用户访问端口 | agent 分配 `20000-20299` 中的 gateway 端口 | Webtop/KasmVNC `3001` |
|
||||
| workspace | `/workspaces/hermes/user-{user_id}/instance-{instance_id}` | `/config/.hermes` |
|
||||
| 典型实现参考 | 按 OpenClaw Lite 的 Runtime Pod Agent V2 做 | 按当前 Hermes Webtop guide 做 |
|
||||
|
||||
@@ -56,7 +56,7 @@ Lite agent 是共享 Pod 内的控制进程,不是某个用户实例内部的
|
||||
- 校验 `X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN}`。
|
||||
- 实现 `GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{gateway_id}`、`POST /v1/drain`。
|
||||
- 向 `${CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/*` 注册、heartbeat、metrics、gateway 状态和 skill inventory。
|
||||
- 管理本 Pod 内多个 Hermes gateway 子进程,端口默认从 `20000-20099` 分配。
|
||||
- 管理本 Pod 内多个 Hermes gateway 子进程,端口默认从 `20000-20299` 分配。
|
||||
- 按 `instance_id + generation` 做幂等创建,重复请求不能启动多个进程。
|
||||
- `POST /v1/gateways` 必须快速返回 `starting`,后台继续启动和健康检查。
|
||||
- gateway 健康后再上报 `running`;缺少 LLM token、workspace 越权、端口冲突或 Hermes 配置失败时上报 `error`。
|
||||
@@ -96,7 +96,7 @@ Content-Type: application/json
|
||||
"agent_type": "hermes",
|
||||
"workspace_path": "/workspaces/hermes/user-45/instance-123",
|
||||
"gateway_port_start": 20000,
|
||||
"gateway_port_end": 20099,
|
||||
"gateway_port_end": 20299,
|
||||
"uid": 200123,
|
||||
"gid": 200123,
|
||||
"environment": {
|
||||
@@ -217,7 +217,7 @@ Hermes Pro 验收:
|
||||
| 错误 | 正确做法 |
|
||||
| --- | --- |
|
||||
| 只实现 Pro instance agent,然后期望 Lite 可用 | Lite 必须实现 Runtime Pod Agent V2 |
|
||||
| Lite agent 监听 `3001` | Lite agent 监听 `19090`,gateway 子进程监听 `20000-20099` |
|
||||
| Lite agent 监听 `3001` | Lite agent 监听 `19090`,gateway 子进程监听 `20000-20299` |
|
||||
| Lite 所有实例共用 `/config/.hermes` | Lite 每个实例使用自己的 `/workspaces/hermes/user-{uid}/instance-{id}` |
|
||||
| `POST /v1/gateways` 阻塞到 Hermes gateway 完全启动 | 快速返回 `starting`,后台健康检查后上报 `running` |
|
||||
| 重试 create 时启动多个 gateway | 按 `instance_id + generation` 幂等 |
|
||||
|
||||
@@ -115,7 +115,7 @@ Gateway create request:
|
||||
"user_id": 45,
|
||||
"agent_type": "openclaw",
|
||||
"workspace_path": "/workspaces/openclaw/user-45/instance-123",
|
||||
"port_range": { "start": 20000, "end": 20099 },
|
||||
"port_range": { "start": 20000, "end": 20299 },
|
||||
"uid": 200123,
|
||||
"gid": 200123,
|
||||
"cpu_cores": 2,
|
||||
@@ -136,7 +136,7 @@ Gateway create response:
|
||||
}
|
||||
```
|
||||
|
||||
The agent must allocate a free port inside `20000-20099`, enforce one Linux UID/GID per instance using `200000 + instance_id`, apply cgroup CPU/memory limits, enforce workspace quota, reject symlink escapes inside workspaces, report gateway health, and survive backend replica changes. During gray upgrade, `POST /v1/drain` must reject new gateways and continue reporting existing gateway health until ClawManager migrates them away.
|
||||
The agent must allocate a free port inside `20000-20299`, enforce one Linux UID/GID per instance using `200000 + instance_id`, apply cgroup CPU/memory limits, enforce workspace quota, reject symlink escapes inside workspaces, report gateway health, and survive backend replica changes. During gray upgrade, `POST /v1/drain` must reject new gateways and continue reporting existing gateway health until ClawManager migrates them away.
|
||||
|
||||
## Task 1: Schema, Models, And Repository Interfaces
|
||||
|
||||
@@ -670,7 +670,7 @@ func TestRuntimeAgentClientCreateGateway(t *testing.T) {
|
||||
UserID: 8,
|
||||
AgentType: "openclaw",
|
||||
WorkspacePath: "/workspaces/openclaw/user-8/instance-7",
|
||||
PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099},
|
||||
PortRange: RuntimeAgentPortRange{Start: 20000, End: 20299},
|
||||
UID: 200007,
|
||||
GID: 200007,
|
||||
CPUCores: 2,
|
||||
@@ -931,7 +931,7 @@ const (
|
||||
RuntimeTypeHermes = "hermes"
|
||||
|
||||
RuntimeGatewayPortStart = 20000
|
||||
RuntimeGatewayPortEnd = 20099
|
||||
RuntimeGatewayPortEnd = 20299
|
||||
RuntimePodCapacity = 100
|
||||
RuntimeLinuxIDBase = 200000
|
||||
)
|
||||
@@ -1133,7 +1133,7 @@ func BuildRuntimeDeployment(spec RuntimeDeploymentSpec) *appsv1.Deployment {
|
||||
{Name: "CLAWMANAGER_RUNTIME_TYPE", Value: spec.RuntimeType},
|
||||
{Name: "CLAWMANAGER_AGENT_PORT", Value: "19090"},
|
||||
{Name: "CLAWMANAGER_GATEWAY_PORT_START", Value: "20000"},
|
||||
{Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: "20099"},
|
||||
{Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: "20299"},
|
||||
{Name: "CLAWMANAGER_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken},
|
||||
{Name: "CLAWMANAGER_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken},
|
||||
},
|
||||
@@ -2867,7 +2867,7 @@ Add env:
|
||||
- name: RUNTIME_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: RUNTIME_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
- name: RUNTIME_SCHEDULER_ENABLED
|
||||
value: "true"
|
||||
- name: RUNTIME_AGENT_CONTROL_TOKEN
|
||||
@@ -2926,7 +2926,7 @@ spec:
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_START
|
||||
value: "20000"
|
||||
- name: CLAWMANAGER_GATEWAY_PORT_END
|
||||
value: "20099"
|
||||
value: "20299"
|
||||
volumeMounts:
|
||||
- name: workspaces
|
||||
mountPath: /workspaces
|
||||
@@ -3075,6 +3075,6 @@ Placeholder scan:
|
||||
Type consistency:
|
||||
- Runtime types are consistently `openclaw` and `hermes`.
|
||||
- Runtime Pod capacity is consistently 100.
|
||||
- Gateway port range is consistently `20000-20099`.
|
||||
- Gateway port range is consistently `20000-20299`.
|
||||
- Workspace paths are consistently `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}`.
|
||||
- Runtime user and group IDs are consistently `200000 + instance_id`.
|
||||
|
||||
@@ -65,7 +65,7 @@ backend replicas: 3
|
||||
openclaw runtime replicas: 1
|
||||
hermes runtime replicas: 1
|
||||
gateways per runtime pod: 100
|
||||
gateway port range per pod: 20000-20099
|
||||
gateway port range per pod: 20000-20299
|
||||
scheduler lease renew: 2s
|
||||
scheduler failover: 6-8s
|
||||
agent heartbeat interval: 2s
|
||||
@@ -337,7 +337,7 @@ Reporting expands from one instance to Pod plus many gateways:
|
||||
"pod_ip": "10.42.1.83",
|
||||
"capacity": 100,
|
||||
"used": 37,
|
||||
"port_range": { "start": 20000, "end": 20099 },
|
||||
"port_range": { "start": 20000, "end": 20299 },
|
||||
"version": "openclaw:2.1.0",
|
||||
"gateways": [
|
||||
{
|
||||
@@ -386,7 +386,7 @@ Create gateway request:
|
||||
"user_id": 7,
|
||||
"agent_type": "openclaw",
|
||||
"workspace_path": "/workspaces/openclaw/user-7/instance-42",
|
||||
"port_range": { "start": 20000, "end": 20099 },
|
||||
"port_range": { "start": 20000, "end": 20299 },
|
||||
"env": {
|
||||
"CLAWMANAGER_INSTANCE_ID": "42",
|
||||
"CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm"
|
||||
@@ -413,7 +413,7 @@ Create gateway response:
|
||||
|
||||
Agent port requirements:
|
||||
|
||||
- Pod-local gateway range is `20000-20099`.
|
||||
- Pod-local gateway range is `20000-20299`.
|
||||
- Agent assigns the actual port.
|
||||
- Agent skips conflicts or returns structured errors.
|
||||
- Agent persists enough state to recover the port occupation map after restart.
|
||||
|
||||
@@ -541,7 +541,6 @@ const InstanceListPage: React.FC = () => {
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={20}
|
||||
value={batchCreateCount}
|
||||
onChange={(event) => setBatchCreateCount(Number(event.target.value))}
|
||||
className="app-input mt-1 w-full"
|
||||
|
||||
Reference in New Issue
Block a user