feat:Improve Lite gateway scheduling, proxy stability, and Hermes recovery (#160)
Release / Prepare Scheduled Release (push) Has been skipped
Release / Publish Release (push) Failing after 0s

* feat(runtime): improve Lite gateway scheduling and proxy stability

- Expand the runtime gateway port range to match 100 instances per pod
- Scale runtime deployments based on pending backlog
- Serialize gateway creation per pod and handle starting/creating states
- Clean up missing gateway bindings from runtime agent reports
- Improve Lite gateway proxy token stripping and auth header injection
- Update deployment manifests, docs, and related tests

* fix(runtime): harden Hermes Lite gateway recovery

- Add hashed gateway token aliases for short-term token compatibility
- Fix /resume conversations failing when old sessions use mismatched gateway tokens
- Recover ports only from stale error bindings without affecting active gateways

* feat(runtime): enable bidirectional desktop clipboard and direct proxy

- Enable WebTop/Selkies/Kasm clipboard by default with per-instance overrides
- Document clipboard verification, configuration pitfalls, and IME troubleshooting
- Enable direct desktop proxy in the K8s and K3s manifests

---------

Co-authored-by: litiantian03 <litiantian03@ieisystem.com>
This commit is contained in:
Li-Day-Day
2026-07-13 14:37:03 +08:00
committed by GitHub
parent ecfc19b7a0
commit 58c19856fd
40 changed files with 1729 additions and 178 deletions
+1
View File
@@ -83,3 +83,4 @@ TEAM_PROFILES_LOCAL_TESTING.md
/clawmanager-apply.sh
/clawmanager-team-profiles-test.yaml
/clawmanager-tenant.yaml
.codex
+1 -1
View File
@@ -286,7 +286,7 @@ func Load() (*Config, error) {
HermesImage: getEnv("HERMES_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"),
MaxGatewaysPerPod: getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", 100),
GatewayPortStart: getEnvInt("RUNTIME_GATEWAY_PORT_START", 20000),
GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", 20099),
GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", 20299),
},
ObjectStorage: ObjectStorageConfig{
Endpoint: getEnv("OBJECT_STORAGE_ENDPOINT", ""),
@@ -0,0 +1,14 @@
CREATE TABLE IF NOT EXISTS instance_gateway_token_aliases (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
instance_id INT NOT NULL,
token_hash CHAR(64) NOT NULL,
expires_at TIMESTAMP NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
last_used_at TIMESTAMP NULL,
CONSTRAINT fk_instance_gateway_token_aliases_instance
FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE,
UNIQUE KEY uk_instance_gateway_token_aliases_hash (token_hash),
INDEX idx_instance_gateway_token_aliases_instance (instance_id),
INDEX idx_instance_gateway_token_aliases_expires (expires_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+23
View File
@@ -160,3 +160,26 @@ func TestMigration037AddsTeamWorkflowLedger(t *testing.T) {
}
}
}
func TestMigration038AddsGatewayTokenAliases(t *testing.T) {
raw, err := embeddedMigrations.ReadFile("migrations/038_add_instance_gateway_token_aliases.sql")
if err != nil {
t.Fatalf("read migration 038: %v", err)
}
sql := string(raw)
for _, required := range []string{
"CREATE TABLE IF NOT EXISTS instance_gateway_token_aliases",
"token_hash CHAR(64)",
"expires_at TIMESTAMP NOT NULL",
"last_used_at TIMESTAMP NULL",
"uk_instance_gateway_token_aliases_hash",
"ON DELETE CASCADE",
} {
if !strings.Contains(sql, required) {
t.Fatalf("migration 038 must contain %s", required)
}
}
if strings.Contains(sql, "access_token") {
t.Fatalf("migration 038 must not store raw access tokens")
}
}
@@ -28,7 +28,6 @@ const openclawMinArchiveBytes = 100
const (
defaultWorkspaceArchiveMaxMiB = int64(500)
workspaceArchiveMaxMiBEnv = "CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB"
maxLiteBatchCreateCount = 20
maxLiteBatchDeleteCount = 100
)
@@ -226,7 +225,7 @@ type BatchCreateLiteInstanceTemplate struct {
// BatchCreateLiteInstancesRequest represents a lite batch create request
type BatchCreateLiteInstancesRequest struct {
NamePrefix string `json:"name_prefix" binding:"required,min=1,max=40"`
Count int `json:"count" binding:"required,min=1,max=20"`
Count int `json:"count" binding:"required,min=1"`
StartIndex int `json:"start_index" binding:"omitempty,min=0,max=9999"`
Template *BatchCreateLiteInstanceTemplate `json:"template,omitempty"`
}
@@ -479,9 +478,6 @@ func buildLiteBatchCreateRequests(req BatchCreateLiteInstancesRequest) ([]servic
if count <= 0 {
return nil, nil, fmt.Errorf("count is required")
}
if count > maxLiteBatchCreateCount {
return nil, nil, fmt.Errorf("count must be at most %d", maxLiteBatchCreateCount)
}
prefix := strings.TrimSpace(req.NamePrefix)
if prefix == "" {
@@ -272,7 +272,9 @@ func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) {
if !ok {
return
}
reported := make(map[int]int, len(req.Gateways))
for _, gateway := range req.Gateways {
reported[gateway.InstanceID] = gateway.Generation
binding, err := h.bindingRepo.GetByInstanceID(c.Request.Context(), gateway.InstanceID)
if err != nil {
utils.HandleError(c, err)
@@ -281,20 +283,37 @@ func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) {
if binding == nil || binding.RuntimePodID != podID || binding.Generation != gateway.Generation {
continue
}
state := strings.ToLower(strings.TrimSpace(gateway.State))
switch state {
case "running", "ready", "healthy":
lifecycle := services.NormalizeRuntimeGatewayLifecycle(gateway.State, gateway.ErrorMessage)
if lifecycle.Running {
if err := h.bindingRepo.UpdateRunning(c.Request.Context(), gateway.InstanceID, gateway.Generation, strings.TrimSpace(gateway.GatewayID), gateway.GatewayPort, gateway.GatewayPID); err != nil {
utils.HandleError(c, err)
return
}
default:
if err := h.bindingRepo.UpdateState(c.Request.Context(), gateway.InstanceID, gateway.Generation, state, gateway.ErrorMessage); err != nil {
} else {
if err := h.bindingRepo.UpdateState(c.Request.Context(), gateway.InstanceID, gateway.Generation, lifecycle.BindingState, lifecycle.Message); err != nil {
utils.HandleError(c, err)
return
}
}
if err := h.syncInstanceRuntimeState(c.Request.Context(), gateway, state); err != nil {
if err := h.syncInstanceRuntimeState(c.Request.Context(), gateway, lifecycle); err != nil {
utils.HandleError(c, err)
return
}
}
bindings, err := h.bindingRepo.ListByRuntimePodID(c.Request.Context(), podID)
if err != nil {
utils.HandleError(c, err)
return
}
now := time.Now().UTC()
for _, binding := range bindings {
if _, reportedByInstance := reported[binding.InstanceID]; reportedByInstance {
continue
}
if !h.missingGatewayCleanupEligible(binding, now) {
continue
}
if _, err := h.bindingRepo.DeleteRunningByInstanceIDGenerationAndReleaseSlot(c.Request.Context(), binding.InstanceID, podID, binding.Generation); err != nil {
utils.HandleError(c, err)
return
}
@@ -306,35 +325,21 @@ func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) {
utils.Success(c, http.StatusOK, "Runtime gateway report accepted", nil)
}
func (h *RuntimeAgentHandler) syncInstanceRuntimeState(ctx context.Context, gateway runtimeAgentGatewayReport, state string) error {
func (h *RuntimeAgentHandler) missingGatewayCleanupEligible(binding models.InstanceRuntimeBinding, now time.Time) bool {
if h == nil || h.cfg.HeartbeatTimeout <= 0 || binding.LastHealthAt == nil {
return false
}
if !strings.EqualFold(strings.TrimSpace(binding.State), services.RuntimeGatewayBindingRunning) {
return false
}
return !binding.LastHealthAt.After(now.Add(-h.cfg.HeartbeatTimeout))
}
func (h *RuntimeAgentHandler) syncInstanceRuntimeState(ctx context.Context, gateway runtimeAgentGatewayReport, lifecycle services.RuntimeGatewayLifecycleState) error {
if h.instanceRepo == nil {
return nil
}
instanceState, message := instanceRuntimeStateFromGatewayReport(state, gateway.ErrorMessage)
return h.instanceRepo.UpdateRuntimeState(ctx, gateway.InstanceID, instanceState, gateway.Generation, message)
}
func instanceRuntimeStateFromGatewayReport(state string, errorMessage *string) (string, *string) {
normalized := strings.ToLower(strings.TrimSpace(state))
switch normalized {
case "running", "ready", "healthy":
return "running", nil
case "error", "failed", "failure", "errored":
if errorMessage != nil && strings.TrimSpace(*errorMessage) != "" {
msg := strings.TrimSpace(*errorMessage)
return "error", &msg
}
msg := "runtime gateway reported " + normalized
return "error", &msg
case "stopped", "deleted":
return "stopped", nil
default:
msg := "runtime gateway starting"
if errorMessage != nil && strings.TrimSpace(*errorMessage) != "" {
msg = strings.TrimSpace(*errorMessage)
}
return "creating", &msg
}
return h.instanceRepo.UpdateRuntimeState(ctx, gateway.InstanceID, lifecycle.InstanceState, gateway.Generation, lifecycle.Message)
}
func (h *RuntimeAgentHandler) ReportSkills(c *gin.Context) {
@@ -5,6 +5,7 @@ import (
"context"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"time"
@@ -211,6 +212,9 @@ func TestRuntimeAgentHandlerGatewayReportOnlyUpdatesCurrentPodBinding(t *testing
if bindingRepo.updateRunningCalls != 1 {
t.Fatalf("UpdateRunning calls = %d, want 1", bindingRepo.updateRunningCalls)
}
if len(bindingRepo.deletedInstanceIDs) != 0 {
t.Fatalf("stale generation report deleted instances %v", bindingRepo.deletedInstanceIDs)
}
}
func TestRuntimeAgentHandlerGatewayReportSyncsInstanceRuntimeState(t *testing.T) {
@@ -260,6 +264,67 @@ func TestRuntimeAgentHandlerGatewayReportSyncsInstanceRuntimeState(t *testing.T)
}
}
func TestRuntimeAgentHandlerGatewayReportDeletesMissingCurrentPodBinding(t *testing.T) {
gin.SetMode(gin.TestMode)
lastHealthAt := time.Now().UTC().Add(-time.Minute)
bindingRepo := &runtimeAgentHandlerBindingRepo{
bindings: map[int]*models.InstanceRuntimeBinding{
10: {InstanceID: 10, RuntimePodID: 9, Generation: 2, State: "running", LastHealthAt: &lastHealthAt},
11: {InstanceID: 11, RuntimePodID: 99, Generation: 2, State: "running"},
12: {InstanceID: 12, RuntimePodID: 9, Generation: 2, State: "error"},
},
}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{})
router := gin.New()
router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/gateways/report", bytes.NewBufferString(`{"pod_id":9,"gateways":[]}`))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-ClawManager-Agent-Token", "secret")
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
if got, want := bindingRepo.deletedInstanceIDs, []int{10}; !reflect.DeepEqual(got, want) {
t.Fatalf("deleted instance IDs = %v, want %v", got, want)
}
if got, want := bindingRepo.deletedRuntimePodIDs, []int64{9}; !reflect.DeepEqual(got, want) {
t.Fatalf("released runtime pod IDs = %v, want %v", got, want)
}
}
func TestRuntimeAgentHandlerGatewayReportDoesNotDeleteFreshBindingFromFirstEmptySnapshot(t *testing.T) {
gin.SetMode(gin.TestMode)
lastHealthAt := time.Now().UTC()
bindingRepo := &runtimeAgentHandlerBindingRepo{
bindings: map[int]*models.InstanceRuntimeBinding{
10: {InstanceID: 10, RuntimePodID: 9, Generation: 3, State: "running", LastHealthAt: &lastHealthAt},
},
}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{})
router := gin.New()
router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/gateways/report", bytes.NewBufferString(`{"pod_id":9,"gateways":[]}`))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-ClawManager-Agent-Token", "secret")
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
}
if len(bindingRepo.deletedInstanceIDs) != 0 {
t.Fatalf("fresh binding deleted by first empty snapshot: %v", bindingRepo.deletedInstanceIDs)
}
if bindingRepo.bindings[10] == nil {
t.Fatal("fresh binding was removed")
}
}
type runtimeAgentHandlerPodRepo struct {
updatedPodID int64
lastMetrics repository.RuntimePodMetricsUpdate
@@ -334,9 +399,11 @@ func (r *runtimeAgentHandlerPodRepo) UpdateMetrics(ctx context.Context, podID in
}
type runtimeAgentHandlerBindingRepo struct {
updateRunningCalls int
updateStateCalls int
bindings map[int]*models.InstanceRuntimeBinding
updateRunningCalls int
updateStateCalls int
bindings map[int]*models.InstanceRuntimeBinding
deletedInstanceIDs []int
deletedRuntimePodIDs []int64
}
func (r *runtimeAgentHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
@@ -352,7 +419,14 @@ func (r *runtimeAgentHandlerBindingRepo) GetRunningByInstanceID(ctx context.Cont
}
func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
return nil, nil
bindings := make([]models.InstanceRuntimeBinding, 0, len(r.bindings))
for _, binding := range r.bindings {
if binding.RuntimePodID != runtimePodID {
continue
}
bindings = append(bindings, *binding)
}
return bindings, nil
}
func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) {
@@ -369,14 +443,31 @@ func (r *runtimeAgentHandlerBindingRepo) UpdateState(ctx context.Context, instan
return nil
}
func (r *runtimeAgentHandlerBindingRepo) DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error) {
return 0, nil
}
func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
return nil
}
func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
r.deletedInstanceIDs = append(r.deletedInstanceIDs, instanceID)
r.deletedRuntimePodIDs = append(r.deletedRuntimePodIDs, runtimePodID)
delete(r.bindings, instanceID)
return nil
}
func (r *runtimeAgentHandlerBindingRepo) DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error) {
binding := r.bindings[instanceID]
if binding == nil || binding.RuntimePodID != runtimePodID || binding.Generation != generation || binding.State != "running" {
return false, nil
}
r.deletedInstanceIDs = append(r.deletedInstanceIDs, instanceID)
r.deletedRuntimePodIDs = append(r.deletedRuntimePodIDs, runtimePodID)
delete(r.bindings, instanceID)
return true, nil
}
type runtimeAgentHandlerInstanceRepo struct {
statusByID map[int]string
generationByID map[int]int
@@ -429,12 +429,18 @@ func (r *runtimePoolHandlerBindingRepo) UpdateRunning(ctx context.Context, insta
func (r *runtimePoolHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
return nil
}
func (r *runtimePoolHandlerBindingRepo) DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error) {
return 0, nil
}
func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
return nil
}
func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
return nil
}
func (r *runtimePoolHandlerBindingRepo) DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error) {
return false, nil
}
type runtimePoolHandlerRolloutRepo struct {
created *models.RuntimeRollout
@@ -2,7 +2,9 @@ package repository
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"strings"
@@ -72,19 +74,87 @@ func (r *instanceRepository) GetByID(id int) (*models.Instance, error) {
return &instance, nil
}
// GetByAccessToken gets an instance by its lifecycle gateway token.
// GetByAccessToken gets an instance by its lifecycle gateway token or a
// still-valid historical token alias.
func (r *instanceRepository) GetByAccessToken(accessToken string) (*models.Instance, error) {
token := strings.TrimSpace(accessToken)
if token == "" {
return nil, nil
}
var instance models.Instance
err := r.sess.Collection("instances").Find(db.Cond{"access_token": accessToken}).One(&instance)
if err != nil {
err := r.sess.Collection("instances").Find(db.Cond{"access_token": token}).One(&instance)
if err == nil {
return &instance, nil
}
if err != db.ErrNoMoreRows {
return nil, fmt.Errorf("failed to get instance by access token: %w", err)
}
return r.getByGatewayTokenAlias(token)
}
func (r *instanceRepository) getByGatewayTokenAlias(accessToken string) (*models.Instance, error) {
tokenHash := gatewayTokenHash(accessToken)
if tokenHash == "" {
return nil, nil
}
var instance models.Instance
now := time.Now().UTC()
iter := r.sess.SQL().Iterator(`
SELECT i.*
FROM instances i
JOIN instance_gateway_token_aliases a ON a.instance_id = i.id
WHERE a.token_hash = ? AND a.expires_at > ?
LIMIT 1
`, tokenHash, now)
defer iter.Close()
if err := iter.One(&instance); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get instance by access token: %w", err)
return nil, fmt.Errorf("failed to get instance by gateway token alias: %w", err)
}
if _, err := r.sess.SQL().Exec(`
UPDATE instance_gateway_token_aliases
SET last_used_at = ?, updated_at = ?
WHERE token_hash = ?
`, now, now, tokenHash); err != nil {
return nil, fmt.Errorf("failed to mark gateway token alias used: %w", err)
}
return &instance, nil
}
func (r *instanceRepository) UpsertGatewayTokenAlias(ctx context.Context, instanceID int, accessToken string, expiresAt time.Time) error {
tokenHash := gatewayTokenHash(accessToken)
if instanceID <= 0 || tokenHash == "" || expiresAt.IsZero() {
return nil
}
now := time.Now().UTC()
_, err := r.sess.SQL().ExecContext(ctx, `
INSERT INTO instance_gateway_token_aliases (instance_id, token_hash, expires_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
instance_id = VALUES(instance_id),
expires_at = GREATEST(expires_at, VALUES(expires_at)),
updated_at = VALUES(updated_at)
`, instanceID, tokenHash, expiresAt.UTC(), now, now)
if err != nil {
return fmt.Errorf("failed to upsert gateway token alias: %w", err)
}
return nil
}
func gatewayTokenHash(accessToken string) string {
token := strings.TrimSpace(accessToken)
if token == "" {
return ""
}
sum := sha256.Sum256([]byte(token))
return hex.EncodeToString(sum[:])
}
func (r *instanceRepository) GetByAgentBootstrapToken(bootstrapToken string) (*models.Instance, error) {
var instance models.Instance
err := r.sess.Collection("instances").Find(db.Cond{"agent_bootstrap_token": bootstrapToken}).One(&instance)
@@ -53,6 +53,21 @@ func TestBuildV2SchedulerInstanceQueryDefaultsLimit(t *testing.T) {
}
}
func TestGatewayTokenHashTrimsAndDoesNotExposeToken(t *testing.T) {
hash := gatewayTokenHash(" igt_secret ")
if len(hash) != 64 {
t.Fatalf("hash length = %d, want 64", len(hash))
}
if strings.Contains(hash, "igt_secret") {
t.Fatalf("hash must not contain raw token")
}
if hash != gatewayTokenHash("igt_secret") {
t.Fatalf("hash should trim surrounding whitespace")
}
if hash == gatewayTokenHash("igt_other") {
t.Fatalf("different tokens should produce different hashes")
}
}
func normalizeSQLForTest(query string) string {
return strings.Join(strings.Fields(query), " ")
}
@@ -20,8 +20,10 @@ type InstanceRuntimeBindingRepository interface {
ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error)
UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error
UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error
DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error)
DeleteByInstanceID(ctx context.Context, instanceID int) error
DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error
DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error)
}
type instanceRuntimeBindingRepository struct {
@@ -152,6 +154,23 @@ func (r *instanceRuntimeBindingRepository) UpdateState(ctx context.Context, inst
return nil
}
func (r *instanceRuntimeBindingRepository) DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error) {
if err := ctx.Err(); err != nil {
return 0, err
}
res, err := r.sess.SQL().ExecContext(ctx, `
DELETE FROM instance_runtime_bindings
WHERE runtime_pod_id = ? AND gateway_port = ? AND state = 'error'
`, runtimePodID, gatewayPort)
if err != nil {
return 0, fmt.Errorf("failed to delete stale error instance runtime binding for gateway port: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return 0, fmt.Errorf("failed to inspect stale error instance runtime binding delete: %w", err)
}
return affected, nil
}
func (r *instanceRuntimeBindingRepository) getGeneration(ctx context.Context, instanceID int) (int, error) {
var currentGeneration int
row, err := r.sess.SQL().QueryRowContext(ctx, `
@@ -211,3 +230,36 @@ func (r *instanceRuntimeBindingRepository) DeleteByInstanceIDAndReleaseSlot(ctx
return nil
}, nil)
}
func (r *instanceRuntimeBindingRepository) DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
deleted := false
err := r.sess.TxContext(ctx, func(tx db.Session) error {
res, err := tx.SQL().ExecContext(ctx, `
DELETE FROM instance_runtime_bindings
WHERE instance_id = ? AND runtime_pod_id = ? AND generation = ? AND state = 'running'
`, instanceID, runtimePodID, generation)
if err != nil {
return fmt.Errorf("failed to delete current running instance runtime binding: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("failed to inspect current running instance runtime binding delete: %w", err)
}
if affected == 0 {
return nil
}
if _, err := tx.SQL().ExecContext(ctx, `
UPDATE runtime_pods
SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ?
WHERE id = ?
`, time.Now().UTC(), runtimePodID); err != nil {
return fmt.Errorf("failed to release runtime pod slot: %w", err)
}
deleted = true
return nil
}, nil)
return deleted, err
}
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"os"
"strings"
"sync"
"time"
@@ -111,6 +112,19 @@ func (s *InstanceAccessService) ValidateToken(token string) (*AccessToken, error
return nil, err
}
func (s *InstanceAccessService) IsInstanceAccessToken(token string) bool {
value := strings.TrimSpace(token)
if value == "" || strings.Count(value, ".") != 2 {
return false
}
parsed, _, err := jwt.NewParser().ParseUnverified(value, &instanceAccessClaims{})
if err != nil {
return false
}
claims, ok := parsed.Claims.(*instanceAccessClaims)
return ok && claims.TokenType == "instance_access" && strings.TrimSpace(claims.AccessURL) != ""
}
func (s *InstanceAccessService) validateSignedToken(token string) (*AccessToken, error) {
parsed, err := jwt.ParseWithClaims(token, &instanceAccessClaims{}, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
+16 -7
View File
@@ -37,9 +37,12 @@ func TestBuildInstancePodEnvAppliesOverridesAfterDefaults(t *testing.T) {
t.Setenv("K8S_NAMESPACE", "")
raw, err := marshalEnvironmentOverrides(map[string]string{
"SUBFOLDER": "/custom-proxy",
"KASM_SVC_ACCEPT_CUT_TEXT": "-AcceptCutText 1",
"CUSTOM": "enabled",
"SUBFOLDER": "/custom-proxy",
"KASM_SVC_ACCEPT_CUT_TEXT": "-AcceptCutText 1",
"SELKIES_CLIPBOARD_ENABLED": "true|locked",
"SELKIES_CLIPBOARD_IN_ENABLED": "true|locked",
"SELKIES_CLIPBOARD_OUT_ENABLED": "true|locked",
"CUSTOM": "enabled",
})
if err != nil {
t.Fatalf("marshalEnvironmentOverrides returned error: %v", err)
@@ -51,10 +54,7 @@ func TestBuildInstancePodEnvAppliesOverridesAfterDefaults(t *testing.T) {
EnvironmentOverridesJSON: raw,
}
env, err := buildInstancePodEnv(instance, map[string]string{
"TITLE": "ClawManager Webtop",
"SUBFOLDER": "/",
}, nil, nil)
env, err := buildInstancePodEnv(instance, defaultWebtopDesktopEnv("ClawManager Webtop"), nil, nil)
if err != nil {
t.Fatalf("buildInstancePodEnv returned error: %v", err)
}
@@ -65,6 +65,15 @@ func TestBuildInstancePodEnvAppliesOverridesAfterDefaults(t *testing.T) {
if env["KASM_SVC_ACCEPT_CUT_TEXT"] != "-AcceptCutText 1" {
t.Fatalf("expected clipboard policy override to win, got %q", env["KASM_SVC_ACCEPT_CUT_TEXT"])
}
for _, key := range []string{
"SELKIES_CLIPBOARD_ENABLED",
"SELKIES_CLIPBOARD_IN_ENABLED",
"SELKIES_CLIPBOARD_OUT_ENABLED",
} {
if got := env[key]; got != "true|locked" {
t.Fatalf("expected %s override to win, got %q", key, got)
}
}
if env["CUSTOM"] != "enabled" {
t.Fatalf("expected custom environment variable to be merged")
}
@@ -144,9 +144,11 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
return err
}
// Copy query parameters (excluding token)
managedGatewayToken := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType)
// Copy query parameters, excluding ClawManager-owned proxy/gateway tokens.
queryParams := r.URL.Query()
removeProxyAccessTokenQuery(queryParams, token)
s.removeProxyAccessTokenQuery(queryParams, token, managedGatewayToken)
if len(queryParams) > 0 {
targetURL.RawQuery = queryParams.Encode()
}
@@ -172,9 +174,7 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
proxyReq.Header.Set("X-Forwarded-Host", r.Host)
proxyReq.Header.Set("X-Forwarded-Proto", requestScheme(r))
proxyReq.Header.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID))
if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" {
proxyReq.Header.Set("Authorization", "Bearer "+token)
}
setManagedRuntimeGatewayAuthHeaders(proxyReq.Header, managedGatewayToken)
if shouldRewriteHTML {
proxyReq.Header.Del("Accept-Encoding")
}
@@ -270,9 +270,11 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in
return err
}
// Copy query parameters (excluding token)
managedGatewayToken := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType)
// Copy query parameters, excluding ClawManager-owned proxy/gateway tokens.
queryParams := r.URL.Query()
removeProxyAccessTokenQuery(queryParams, token)
s.removeProxyAccessTokenQuery(queryParams, token, managedGatewayToken)
if len(queryParams) > 0 {
targetURL.RawQuery = queryParams.Encode()
}
@@ -293,8 +295,8 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in
upstreamHeader.Set("X-Forwarded-Host", r.Host)
upstreamHeader.Set("X-Forwarded-Proto", requestScheme(r))
upstreamHeader.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID))
if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" {
upstreamHeader.Set("Authorization", "Bearer "+token)
setManagedRuntimeGatewayAuthHeaders(upstreamHeader, managedGatewayToken)
if managedGatewayToken != "" {
upstreamHeader.Set("Origin", s.openClawWebSocketOrigin(targetURL))
}
@@ -389,6 +391,18 @@ func (s *InstanceProxyService) removeHopByHopHeaders(header http.Header) {
}
}
func setManagedRuntimeGatewayAuthHeaders(header http.Header, token string) {
token = strings.TrimSpace(token)
if token == "" {
return
}
header.Set("Authorization", "Bearer "+token)
header.Set("X-Api-Key", token)
header.Set("X-OpenAI-Api-Key", token)
header.Set("OpenAI-Api-Key", token)
header.Set("X-ClawManager-Instance-Token", token)
header.Set("X-ClawManager-LLM-API-Key", token)
}
func (s *InstanceProxyService) managedRuntimeGatewayBearerToken(ctx context.Context, instanceID int, instanceType string) string {
if s == nil || s.instanceRepo == nil {
return ""
@@ -838,14 +852,14 @@ func proxyURLWithPath(instanceID int, targetPath, token string) string {
return fmt.Sprintf("%s?token=%s", raw, url.QueryEscape(token))
}
func removeProxyAccessTokenQuery(query url.Values, accessToken string) {
func (s *InstanceProxyService) removeProxyAccessTokenQuery(query url.Values, accessToken, managedGatewayToken string) {
values := query["token"]
if len(values) == 0 {
return
}
filtered := values[:0]
for _, value := range values {
if value != accessToken {
if !s.shouldRemoveProxyTokenQueryValue(value, accessToken, managedGatewayToken) {
filtered = append(filtered, value)
}
}
@@ -856,6 +870,23 @@ func removeProxyAccessTokenQuery(query url.Values, accessToken string) {
query["token"] = filtered
}
func (s *InstanceProxyService) shouldRemoveProxyTokenQueryValue(value, accessToken, managedGatewayToken string) bool {
candidate := strings.TrimSpace(value)
if candidate == "" {
return false
}
if candidate == accessToken || (managedGatewayToken != "" && candidate == managedGatewayToken) {
return true
}
if s != nil && s.accessService != nil && s.accessService.IsInstanceAccessToken(candidate) {
return true
}
return managedGatewayToken != "" && looksLikeManagedInstanceGatewayToken(candidate)
}
func looksLikeManagedInstanceGatewayToken(value string) bool {
return strings.HasPrefix(strings.TrimSpace(value), "igt_")
}
func proxyBaseForRequestPath(requestPath string, instanceID int) string {
prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
path := strings.TrimSpace(requestPath)
@@ -94,6 +94,12 @@ func TestInstanceProxyServiceInjectsInstanceTokenForHermesLite(t *testing.T) {
if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
t.Fatalf("Authorization = %q", got)
}
if got := r.Header.Get("X-Api-Key"); got != instanceToken {
t.Fatalf("X-Api-Key = %q", got)
}
if got := r.Header.Get("X-ClawManager-Instance-Token"); got != instanceToken {
t.Fatalf("X-ClawManager-Instance-Token = %q", got)
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
}))
@@ -139,6 +145,7 @@ func TestInstanceProxyServiceInjectsInstanceTokenForHermesLite(t *testing.T) {
service.httpClient = upstream.Client()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/127/proxy/chat?token="+url.QueryEscape(token.Token), nil)
req.Header.Set("X-Api-Key", "stale-session-key")
rec := httptest.NewRecorder()
if err := service.ProxyRequest(req.Context(), 127, token.Token, rec, req); err != nil {
@@ -284,6 +291,94 @@ func TestInstanceProxyServicePreservesHermesRuntimeQueryToken(t *testing.T) {
}
}
func TestInstanceProxyServiceStripsStaleHermesGatewayQueryToken(t *testing.T) {
instanceToken := "igt_hermes_instance"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/ws" {
t.Fatalf("unexpected upstream path %s", r.URL.Path)
}
if got := r.URL.RawQuery; got != "" {
t.Fatalf("upstream query = %q, want empty", got)
}
if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
t.Fatalf("Authorization = %q", got)
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
}))
defer upstream.Close()
podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
workspacePath := "/workspaces/hermes/user-45/instance-132"
instanceRepo := newV2LifecycleInstanceRepo()
instanceRepo.byID[132] = &models.Instance{
ID: 132,
UserID: 45,
Type: "hermes",
RuntimeType: "gateway",
InstanceMode: InstanceModeLite,
Status: "running",
AccessToken: &instanceToken,
WorkspacePath: &workspacePath,
RuntimeGeneration: 5,
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[132] = &models.InstanceRuntimeBinding{
InstanceID: 132,
RuntimePodID: 15,
GatewayPort: gatewayPort,
State: "running",
Generation: 5,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
15: {ID: 15, PodIP: &podIP, State: "ready"},
},
}
accessService := NewInstanceAccessService()
defer accessService.Stop()
accessToken, err := accessService.GenerateToken(45, 132, "hermes", "/api/v1/instances/132/proxy/chat/", "", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
service.instanceRepo = instanceRepo
service.bindingRepo = bindingRepo
service.runtimePodRepo = podRepo
service.httpClient = upstream.Client()
req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/132/proxy/api/ws?token=igt_stale_gateway", nil)
rec := httptest.NewRecorder()
if err := service.ProxyRequest(req.Context(), 132, accessToken.Token, rec, req); err != nil {
t.Fatalf("ProxyRequest returned error: %v", err)
}
if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
}
}
func TestInstanceProxyServiceStripsStaleProxyAccessTokenQuery(t *testing.T) {
accessService := NewInstanceAccessService()
defer accessService.Stop()
currentToken, err := accessService.GenerateToken(45, 133, "hermes", "/api/v1/instances/133/proxy/chat/", "", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken current returned error: %v", err)
}
oldToken, err := accessService.GenerateToken(45, 133, "hermes", "/api/v1/instances/133/proxy/chat/", "", 3000, time.Hour)
if err != nil {
t.Fatalf("GenerateToken old returned error: %v", err)
}
service := NewInstanceProxyService(accessService)
query := url.Values{"token": []string{oldToken.Token, "hermes-session-token"}}
service.removeProxyAccessTokenQuery(query, currentToken.Token, "igt_hermes_instance")
got := query["token"]
if len(got) != 1 || got[0] != "hermes-session-token" {
t.Fatalf("token query = %#v, want only Hermes session token", got)
}
}
func TestInstanceProxyServiceRewritesHermesLiteHTMLBase(t *testing.T) {
instanceToken := "igt_hermes_instance"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+10 -6
View File
@@ -17,8 +17,9 @@ type InstanceRuntimeConfig struct {
}
const (
kasmClipboardSendDisabled = "-SendCutText 0"
kasmClipboardAcceptDisabled = "-AcceptCutText 0"
kasmClipboardSendEnabled = "-SendCutText 1"
kasmClipboardAcceptEnabled = "-AcceptCutText 1"
selkiesClipboardEnabled = "true|locked"
)
func buildRuntimeConfig(instanceType, osType, osVersion string, registry, tag *string) InstanceRuntimeConfig {
@@ -117,10 +118,13 @@ func defaultEnvForInstanceType(instanceType string) map[string]string {
func defaultWebtopDesktopEnv(title string) map[string]string {
return map[string]string{
"TITLE": title,
"SUBFOLDER": "/",
"KASM_SVC_SEND_CUT_TEXT": kasmClipboardSendDisabled,
"KASM_SVC_ACCEPT_CUT_TEXT": kasmClipboardAcceptDisabled,
"TITLE": title,
"SUBFOLDER": "/",
"KASM_SVC_SEND_CUT_TEXT": kasmClipboardSendEnabled,
"KASM_SVC_ACCEPT_CUT_TEXT": kasmClipboardAcceptEnabled,
"SELKIES_CLIPBOARD_ENABLED": selkiesClipboardEnabled,
"SELKIES_CLIPBOARD_IN_ENABLED": selkiesClipboardEnabled,
"SELKIES_CLIPBOARD_OUT_ENABLED": selkiesClipboardEnabled,
}
}
@@ -37,24 +37,39 @@ func TestBuildRuntimeConfig_HermesUsesWebtopDefaults(t *testing.T) {
if config.Env["SUBFOLDER"] != "/" {
t.Fatalf("expected Hermes default SUBFOLDER /, got %q", config.Env["SUBFOLDER"])
}
if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendDisabled {
t.Fatalf("expected Hermes to disable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"])
if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendEnabled {
t.Fatalf("expected Hermes to enable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"])
}
if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptDisabled {
t.Fatalf("expected Hermes to disable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"])
if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptEnabled {
t.Fatalf("expected Hermes to enable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"])
}
assertSelkiesClipboardEnabled(t, config.Env)
if !usesWebtopImage("hermes") {
t.Fatalf("expected Hermes to use webtop proxy behavior")
}
}
func TestBuildRuntimeConfig_OpenClawDisablesKasmClipboardSync(t *testing.T) {
func TestBuildRuntimeConfig_OpenClawEnablesDesktopClipboardSync(t *testing.T) {
config := buildRuntimeConfig("openclaw", "openclaw", "latest", nil, nil)
if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendDisabled {
t.Fatalf("expected OpenClaw to disable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"])
if config.Env["KASM_SVC_SEND_CUT_TEXT"] != kasmClipboardSendEnabled {
t.Fatalf("expected OpenClaw to enable outbound clipboard sync, got %q", config.Env["KASM_SVC_SEND_CUT_TEXT"])
}
if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptDisabled {
t.Fatalf("expected OpenClaw to disable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"])
if config.Env["KASM_SVC_ACCEPT_CUT_TEXT"] != kasmClipboardAcceptEnabled {
t.Fatalf("expected OpenClaw to enable inbound clipboard sync, got %q", config.Env["KASM_SVC_ACCEPT_CUT_TEXT"])
}
assertSelkiesClipboardEnabled(t, config.Env)
}
func assertSelkiesClipboardEnabled(t *testing.T, env map[string]string) {
t.Helper()
for _, key := range []string{
"SELKIES_CLIPBOARD_ENABLED",
"SELKIES_CLIPBOARD_IN_ENABLED",
"SELKIES_CLIPBOARD_OUT_ENABLED",
} {
if got := env[key]; got != selkiesClipboardEnabled {
t.Fatalf("expected %s=%q, got %q", key, selkiesClipboardEnabled, got)
}
}
}
+37 -1
View File
@@ -220,6 +220,14 @@ type instanceService struct {
networkPolicyService *k8s.NetworkPolicyService
}
const (
defaultGatewayTokenAliasTTL = 7 * 24 * time.Hour
gatewayTokenAliasTTLEnv = "CLAWMANAGER_GATEWAY_TOKEN_ALIAS_TTL_HOURS"
)
type gatewayTokenAliasRecorder interface {
UpsertGatewayTokenAlias(ctx context.Context, instanceID int, accessToken string, expiresAt time.Time) error
}
type gatewayModelInjection struct {
defaultModel string
modelsJSON string
@@ -987,7 +995,9 @@ func (s *instanceService) securityModeForInstance(instanceType string) k8s.PodSe
func (s *instanceService) ensureGatewayToken(instance *models.Instance) (string, error) {
if instance.AccessToken != nil && strings.TrimSpace(*instance.AccessToken) != "" {
return strings.TrimSpace(*instance.AccessToken), nil
token := strings.TrimSpace(*instance.AccessToken)
s.refreshGatewayTokenAlias(instance.ID, token)
return token, nil
}
tokenBytes := make([]byte, 32)
@@ -1002,9 +1012,34 @@ func (s *instanceService) ensureGatewayToken(instance *models.Instance) (string,
return "", fmt.Errorf("failed to persist instance gateway token: %w", err)
}
s.refreshGatewayTokenAlias(instance.ID, token)
return token, nil
}
func (s *instanceService) refreshGatewayTokenAlias(instanceID int, token string) {
recorder, ok := s.instanceRepo.(gatewayTokenAliasRecorder)
if !ok {
return
}
ttl := gatewayTokenAliasTTL()
if ttl <= 0 || strings.TrimSpace(token) == "" || instanceID <= 0 {
return
}
if err := recorder.UpsertGatewayTokenAlias(context.Background(), instanceID, token, time.Now().UTC().Add(ttl)); err != nil {
fmt.Printf("Warning: failed to refresh gateway token alias for instance %d: %v\n", instanceID, err)
}
}
func gatewayTokenAliasTTL() time.Duration {
value := optionalIntEnv(gatewayTokenAliasTTLEnv)
if value == nil {
return defaultGatewayTokenAliasTTL
}
if *value <= 0 {
return 0
}
return time.Duration(*value) * time.Hour
}
func (s *instanceService) buildGatewayEnv(instance *models.Instance) (map[string]string, error) {
if instance == nil || instance.AccessToken == nil || strings.TrimSpace(*instance.AccessToken) == "" {
return map[string]string{}, nil
@@ -1024,6 +1059,7 @@ func (s *instanceService) buildGatewayEnv(instance *models.Instance) (map[string
}
token := strings.TrimSpace(*instance.AccessToken)
s.refreshGatewayTokenAlias(instance.ID, token)
return map[string]string{
"CLAWMANAGER_LLM_BASE_URL": baseURL,
"CLAWMANAGER_LLM_API_KEY": token,
@@ -1,12 +1,31 @@
package services
import (
"context"
"strings"
"testing"
"time"
"clawreef/internal/models"
)
func TestGatewayTokenAliasTTLConfig(t *testing.T) {
t.Setenv(gatewayTokenAliasTTLEnv, "")
if got := gatewayTokenAliasTTL(); got != defaultGatewayTokenAliasTTL {
t.Fatalf("default gateway token alias TTL = %v, want %v", got, defaultGatewayTokenAliasTTL)
}
t.Setenv(gatewayTokenAliasTTLEnv, "2")
if got := gatewayTokenAliasTTL(); got != 2*time.Hour {
t.Fatalf("configured gateway token alias TTL = %v, want 2h", got)
}
t.Setenv(gatewayTokenAliasTTLEnv, "0")
if got := gatewayTokenAliasTTL(); got != 0 {
t.Fatalf("disabled gateway token alias TTL = %v, want 0", got)
}
}
type stubLLMModelRepository struct {
active []models.LLMModel
}
@@ -109,8 +128,50 @@ func TestBuildGatewayEnvEnsuresMissingGatewayToken(t *testing.T) {
if got := instanceRepo.updated[68]; got == nil || got.AccessToken == nil || *got.AccessToken != *instance.AccessToken {
t.Fatalf("repository update did not persist provisioned token: %#v", instanceRepo.updated[68])
}
alias, ok := instanceRepo.aliases[68]
if !ok {
t.Fatal("repository did not record provisioned token as a gateway alias")
}
if alias.token != *instance.AccessToken {
t.Fatalf("gateway alias token = %q, want provisioned token %q", alias.token, *instance.AccessToken)
}
if alias.expiresAt.Before(time.Now().Add(6 * 24 * time.Hour)) {
t.Fatalf("gateway alias expiry = %v, want roughly the default compatibility window", alias.expiresAt)
}
}
func TestBuildGatewayEnvRecordsExistingGatewayTokenAlias(t *testing.T) {
t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm")
token := "igt_existing_token"
instanceRepo := &stubGatewayEnvInstanceRepository{fakeRuntimeInstanceRepo: newFakeRuntimeInstanceRepo()}
service := &instanceService{
instanceRepo: instanceRepo,
llmModelRepo: &stubLLMModelRepository{
active: []models.LLMModel{{DisplayName: "auto"}},
},
}
env, err := service.BuildGatewayEnv(&models.Instance{
ID: 69,
UserID: 1,
Type: "hermes",
AccessToken: &token,
})
if err != nil {
t.Fatalf("BuildGatewayEnv returned error: %v", err)
}
if env["CLAWMANAGER_LLM_API_KEY"] != token {
t.Fatalf("CLAWMANAGER_LLM_API_KEY = %q, want existing token", env["CLAWMANAGER_LLM_API_KEY"])
}
alias, ok := instanceRepo.aliases[69]
if !ok {
t.Fatal("repository did not record existing token as a gateway alias")
}
if alias.token != token {
t.Fatalf("gateway alias token = %q, want existing token %q", alias.token, token)
}
}
func TestBuildGatewayEnvMergesEnvironmentOverrides(t *testing.T) {
t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm")
token := "igt_test_token"
@@ -169,9 +230,15 @@ func TestBuildGatewayEnvSkipsUnmanagedRuntime(t *testing.T) {
}
}
type gatewayTokenAliasRecord struct {
token string
expiresAt time.Time
}
type stubGatewayEnvInstanceRepository struct {
*fakeRuntimeInstanceRepo
updated map[int]*models.Instance
aliases map[int]gatewayTokenAliasRecord
}
func (r *stubGatewayEnvInstanceRepository) Update(instance *models.Instance) error {
@@ -183,6 +250,13 @@ func (r *stubGatewayEnvInstanceRepository) Update(instance *models.Instance) err
return nil
}
func (r *stubGatewayEnvInstanceRepository) UpsertGatewayTokenAlias(ctx context.Context, instanceID int, accessToken string, expiresAt time.Time) error {
if r.aliases == nil {
r.aliases = map[int]gatewayTokenAliasRecord{}
}
r.aliases[instanceID] = gatewayTokenAliasRecord{token: accessToken, expiresAt: expiresAt}
return nil
}
func TestBuildAgentEnvInjectsHermesAgentConfig(t *testing.T) {
t.Setenv("CLAWMANAGER_AGENT_CONTROL_BASE_URL", "http://agent-control.example")
@@ -25,7 +25,7 @@ const (
runtimeWorkspaceVolume = "workspaces"
defaultWorkspaceMount = "/workspaces"
defaultGatewayPortStart = 20000
defaultGatewayPortEnd = 20099
defaultGatewayPortEnd = 20299
hermesTUIDir = "/usr/local/lib/hermes-agent/ui-tui"
)
@@ -51,7 +51,7 @@ func TestBuildRuntimeDeployment(t *testing.T) {
requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "openclaw")
requireEnv(t, container, "CLAWMANAGER_AGENT_PORT", "19090")
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "20000")
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "20099")
requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "20299")
requireEnv(t, container, "CLAWMANAGER_AGENT_CONTROL_TOKEN", "control-secret")
requireEnv(t, container, "CLAWMANAGER_AGENT_REPORT_TOKEN", "report-secret")
@@ -37,7 +37,7 @@ func TestRuntimeAgentClientCreateGateway(t *testing.T) {
UserID: 8,
AgentType: "openclaw",
WorkspacePath: "/workspaces/openclaw/user-8/instance-7",
PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099},
PortRange: RuntimeAgentPortRange{Start: RuntimeGatewayPortStart, End: RuntimeGatewayPortEnd},
UID: 200007,
GID: 200007,
CPUCores: 2,
@@ -18,9 +18,12 @@ const (
RuntimeBackendShell = "shell"
RuntimeGatewayPortStart = 20000
RuntimeGatewayPortEnd = 20099
RuntimePodCapacity = 100
RuntimeLinuxIDBase = 200000
// OpenClaw gateway slots reserve adjacent service ports in addition to the
// public gateway port. Capacity is instance count, not raw port count.
RuntimeGatewayPortsPerInstance = 3
RuntimeGatewayPortEnd = RuntimeGatewayPortStart + RuntimePodCapacity*RuntimeGatewayPortsPerInstance - 1
RuntimeLinuxIDBase = 200000
)
func NormalizeV2RuntimeType(instanceType string) (string, bool) {
@@ -36,6 +36,19 @@ func TestRuntimeLinuxID(t *testing.T) {
}
}
func TestRuntimeGatewayPortRangeMatchesInstanceCapacity(t *testing.T) {
if got, want := RuntimeGatewayPortsPerInstance, 3; got != want {
t.Fatalf("expected %d ports per gateway instance, got %d", want, got)
}
wantEnd := RuntimeGatewayPortStart + RuntimePodCapacity*RuntimeGatewayPortsPerInstance - 1
if RuntimeGatewayPortEnd != wantEnd {
t.Fatalf("gateway port end = %d, want %d", RuntimeGatewayPortEnd, wantEnd)
}
if got := (RuntimeGatewayPortEnd - RuntimeGatewayPortStart + 1) / RuntimeGatewayPortsPerInstance; got != RuntimePodCapacity {
t.Fatalf("gateway port range supports %d instances, want %d", got, RuntimePodCapacity)
}
}
func TestInstanceModeRuntimeTypeMapping(t *testing.T) {
if got, ok := RuntimeTypeForInstanceMode(" lite "); !ok || got != RuntimeBackendGateway {
t.Fatalf("lite runtime type = %q/%v, want gateway/true", got, ok)
@@ -0,0 +1,87 @@
package services
import "strings"
const (
RuntimeGatewayBindingCreating = "creating"
RuntimeGatewayBindingRunning = "running"
RuntimeGatewayBindingError = "error"
RuntimeGatewayBindingStopped = "stopped"
)
type RuntimeGatewayLifecycleState struct {
RawState string
BindingState string
InstanceState string
Running bool
Recognized bool
EventType string
Message *string
}
func NormalizeRuntimeGatewayLifecycle(rawState string, reportedMessage *string) RuntimeGatewayLifecycleState {
raw := strings.ToLower(strings.TrimSpace(rawState))
message := normalizedRuntimeGatewayMessage(reportedMessage)
state := RuntimeGatewayLifecycleState{RawState: raw, Recognized: true}
switch raw {
case "running", "ready", "healthy":
state.BindingState = RuntimeGatewayBindingRunning
state.InstanceState = "running"
state.Running = true
state.EventType = "runtime.instance.running"
case "starting", "creating", "pending":
state.BindingState = RuntimeGatewayBindingCreating
state.InstanceState = "creating"
state.EventType = "runtime.instance.starting"
state.Message = messageOrDefault(message, "runtime gateway starting")
case "error", "failed", "failure", "errored":
state.BindingState = RuntimeGatewayBindingError
state.InstanceState = "error"
state.EventType = "runtime.instance.error"
state.Message = messageOrDefault(message, "runtime gateway reported "+raw)
case "stopped", "deleted":
state.BindingState = RuntimeGatewayBindingStopped
state.InstanceState = "stopped"
state.EventType = "runtime.instance.stopped"
state.Message = message
case "":
state.Recognized = false
state.BindingState = RuntimeGatewayBindingCreating
state.InstanceState = "creating"
state.EventType = "runtime.instance.starting"
state.Message = messageOrDefault(message, "runtime gateway returned an empty state")
default:
state.Recognized = false
state.BindingState = RuntimeGatewayBindingCreating
state.InstanceState = "creating"
state.EventType = "runtime.instance.starting"
state.Message = messageOrDefault(message, "runtime gateway reported unrecognized state: "+raw)
}
return state
}
func (state RuntimeGatewayLifecycleState) CreateAccepted() bool {
if !state.Recognized {
return false
}
return state.BindingState == RuntimeGatewayBindingRunning || state.BindingState == RuntimeGatewayBindingCreating
}
func normalizedRuntimeGatewayMessage(message *string) *string {
if message == nil {
return nil
}
trimmed := strings.TrimSpace(*message)
if trimmed == "" {
return nil
}
return &trimmed
}
func messageOrDefault(message *string, fallback string) *string {
if message != nil {
return message
}
return &fallback
}
@@ -0,0 +1,49 @@
package services
import "testing"
func TestNormalizeRuntimeGatewayLifecycle(t *testing.T) {
tests := []struct {
name string
raw string
bindingState string
instanceState string
running bool
recognized bool
createAccepted bool
}{
{name: "running", raw: "running", bindingState: "running", instanceState: "running", running: true, recognized: true, createAccepted: true},
{name: "ready", raw: "ready", bindingState: "running", instanceState: "running", running: true, recognized: true, createAccepted: true},
{name: "healthy", raw: "healthy", bindingState: "running", instanceState: "running", running: true, recognized: true, createAccepted: true},
{name: "starting", raw: "starting", bindingState: "creating", instanceState: "creating", recognized: true, createAccepted: true},
{name: "creating", raw: "creating", bindingState: "creating", instanceState: "creating", recognized: true, createAccepted: true},
{name: "pending", raw: "pending", bindingState: "creating", instanceState: "creating", recognized: true, createAccepted: true},
{name: "error", raw: "error", bindingState: "error", instanceState: "error", recognized: true},
{name: "stopped", raw: "stopped", bindingState: "stopped", instanceState: "stopped", recognized: true},
{name: "empty", raw: "", bindingState: "creating", instanceState: "creating"},
{name: "unknown", raw: "mystery", bindingState: "creating", instanceState: "creating"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NormalizeRuntimeGatewayLifecycle(tt.raw, nil)
if got.BindingState != tt.bindingState || got.InstanceState != tt.instanceState || got.Running != tt.running || got.Recognized != tt.recognized {
t.Fatalf("state = %+v", got)
}
if got.CreateAccepted() != tt.createAccepted {
t.Fatalf("CreateAccepted = %t, want %t", got.CreateAccepted(), tt.createAccepted)
}
if !tt.running && got.Message == nil && tt.instanceState != "stopped" {
t.Fatal("non-running state is missing a diagnostic message")
}
})
}
}
func TestNormalizeRuntimeGatewayLifecyclePreservesReportedError(t *testing.T) {
message := "gateway port is not listening"
got := NormalizeRuntimeGatewayLifecycle("failed", &message)
if got.Message == nil || *got.Message != message {
t.Fatalf("message = %#v, want %q", got.Message, message)
}
}
+237 -53
View File
@@ -7,6 +7,7 @@ import (
"log"
"strconv"
"strings"
"sync"
"time"
"clawreef/internal/models"
@@ -32,11 +33,21 @@ type RuntimeScheduler struct {
gatewayPortEnd int
heartbeatTimeout time.Duration
maxGatewaysPerPod int
gatewayCreateLocksMu sync.Mutex
gatewayCreateLocks map[int64]*sync.Mutex
}
var errRuntimeScaleOutPending = errors.New("runtime scale-out pending")
var (
errRuntimeScaleOutPending = errors.New("runtime scale-out pending")
errRuntimeGatewayStartPending = errors.New("runtime gateway start pending")
)
const runtimeRolloutStaleWindowMultiplier = 3
const (
runtimeRolloutStaleWindowMultiplier = 3
runtimeGatewayStartInFlightLimit = 1
runtimeSchedulerBatchLimit = 1000
)
type RuntimeGatewayEnvBuilder func(*models.Instance) (map[string]string, error)
@@ -108,21 +119,22 @@ func NewRuntimeScheduler(
tick = 2 * time.Second
}
s := &RuntimeScheduler{
instanceRepo: instanceRepo,
podRepo: podRepo,
bindingRepo: bindingRepo,
rolloutRepo: rolloutRepo,
agentClient: agentClient,
events: events,
leader: leader,
deployments: deployments,
tick: tick,
workspaceRoot: "/workspaces",
runtimeNamespace: "clawmanager-system",
gatewayPortStart: RuntimeGatewayPortStart,
gatewayPortEnd: RuntimeGatewayPortEnd,
heartbeatTimeout: 10 * time.Second,
maxGatewaysPerPod: RuntimePodCapacity,
instanceRepo: instanceRepo,
podRepo: podRepo,
bindingRepo: bindingRepo,
rolloutRepo: rolloutRepo,
agentClient: agentClient,
events: events,
leader: leader,
deployments: deployments,
tick: tick,
workspaceRoot: "/workspaces",
runtimeNamespace: "clawmanager-system",
gatewayPortStart: RuntimeGatewayPortStart,
gatewayPortEnd: RuntimeGatewayPortEnd,
heartbeatTimeout: 10 * time.Second,
maxGatewaysPerPod: RuntimePodCapacity,
gatewayCreateLocks: map[int64]*sync.Mutex{},
}
for _, opt := range opts {
opt(s)
@@ -493,7 +505,7 @@ func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
if err := s.reconcileRollouts(ctx); err != nil {
errs = append(errs, err)
}
creating, err := s.instanceRepo.GetV2Creating(ctx, 100)
creating, err := s.instanceRepo.GetV2Creating(ctx, runtimeSchedulerBatchLimit)
if err != nil {
errs = append(errs, err)
} else {
@@ -513,7 +525,7 @@ func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
continue
}
if err := s.assignInstance(ctx, instance); err != nil {
if errors.Is(err, errRuntimeScaleOutPending) {
if errors.Is(err, errRuntimeScaleOutPending) || errors.Is(err, errRuntimeGatewayStartPending) {
continue
}
errs = append(errs, fmt.Errorf("assign creating instance %d: %w", instance.ID, err))
@@ -522,10 +534,13 @@ func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
}
}
desired, err := s.instanceRepo.GetV2DesiredRunning(ctx, 100)
desired, err := s.instanceRepo.GetV2DesiredRunning(ctx, runtimeSchedulerBatchLimit)
if err != nil {
errs = append(errs, err)
} else {
if err := s.scaleOutForPendingBacklog(ctx, desired); err != nil {
errs = append(errs, err)
}
for _, instance := range desired {
if !isSchedulerManagedV2Instance(instance) {
continue
@@ -556,7 +571,7 @@ func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
continue
}
if assignErr := s.assignInstance(ctx, instance); assignErr != nil {
if errors.Is(assignErr, errRuntimeScaleOutPending) {
if errors.Is(assignErr, errRuntimeScaleOutPending) || errors.Is(assignErr, errRuntimeGatewayStartPending) {
continue
}
errs = append(errs, fmt.Errorf("assign desired instance %d: %w", instance.ID, assignErr))
@@ -619,6 +634,152 @@ func (s *RuntimeScheduler) failoverStalePods(ctx context.Context) error {
return errors.Join(errs...)
}
func (s *RuntimeScheduler) lockGatewayCreateForPod(podID int64) func() {
if s == nil {
return func() {}
}
s.gatewayCreateLocksMu.Lock()
if s.gatewayCreateLocks == nil {
s.gatewayCreateLocks = map[int64]*sync.Mutex{}
}
lock := s.gatewayCreateLocks[podID]
if lock == nil {
lock = &sync.Mutex{}
s.gatewayCreateLocks[podID] = lock
}
s.gatewayCreateLocksMu.Unlock()
lock.Lock()
return lock.Unlock
}
func (s *RuntimeScheduler) podCanStartGateway(ctx context.Context, podID int64) (bool, error) {
if s == nil || s.bindingRepo == nil || runtimeGatewayStartInFlightLimit <= 0 {
return true, nil
}
bindings, err := s.bindingRepo.ListByRuntimePodID(ctx, podID)
if err != nil {
return false, fmt.Errorf("list runtime pod %d bindings: %w", podID, err)
}
starting := 0
for _, binding := range bindings {
switch strings.ToLower(strings.TrimSpace(binding.State)) {
case "creating", "starting":
starting++
}
}
return starting < runtimeGatewayStartInFlightLimit, nil
}
func (s *RuntimeScheduler) scaleOutForPendingBacklog(ctx context.Context, instances []models.Instance) error {
if s == nil || s.bindingRepo == nil || s.podRepo == nil || s.deployments == nil || len(instances) == 0 {
return nil
}
backlogByType := map[string]int{}
for _, instance := range instances {
if !isSchedulerManagedV2Instance(instance) {
continue
}
if isRuntimeErrorInstance(instance) && !isRecoverableRuntimeSchedulingError(instance) {
continue
}
runtimeType, ok := schedulerRuntimeType(instance)
if !ok {
continue
}
binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
if err != nil {
return fmt.Errorf("get binding for backlog instance %d: %w", instance.ID, err)
}
if binding == nil {
backlogByType[runtimeType]++
}
}
for runtimeType, backlog := range backlogByType {
if backlog <= 0 {
continue
}
if _, err := s.scaleOutIfBacklogExceedsCapacity(ctx, runtimeType, backlog); err != nil {
return err
}
}
return nil
}
func (s *RuntimeScheduler) scaleOutIfBacklogExceedsCapacity(ctx context.Context, runtimeType string, backlog int) (bool, error) {
if s == nil || s.deployments == nil || s.podRepo == nil || backlog <= 0 {
return false, nil
}
pods, err := s.podRepo.List(ctx, runtimeType)
if err != nil {
return false, fmt.Errorf("list %s runtime pods for backlog scale-out: %w", runtimeType, err)
}
groups := map[string]*runtimeDeploymentCapacity{}
for _, pod := range pods {
if pod.RuntimeType != runtimeType || pod.State != "ready" || pod.Draining || pod.Capacity <= 0 {
continue
}
namespace := strings.TrimSpace(pod.Namespace)
name := strings.TrimSpace(pod.DeploymentName)
if namespace == "" || name == "" {
continue
}
key := namespace + "/" + name
group := groups[key]
if group == nil {
group = &runtimeDeploymentCapacity{namespace: namespace, name: name}
groups[key] = group
}
group.active++
group.capacityTotal += pod.Capacity
group.usedSlots += minInt(pod.UsedSlots, pod.Capacity)
}
var target *runtimeDeploymentCapacity
for _, group := range groups {
if group.active == 0 || group.capacityTotal <= 0 {
continue
}
if target == nil || group.active > target.active {
target = group
}
}
if target == nil {
return false, nil
}
desiredSlots := target.usedSlots + backlog
if desiredSlots <= target.capacityTotal {
return false, nil
}
capacityPerPod := target.capacityTotal / target.active
if capacityPerPod <= 0 {
capacityPerPod = s.maxGatewaysPerPod
}
if capacityPerPod <= 0 {
capacityPerPod = RuntimePodCapacity
}
targetReplicas := (desiredSlots + capacityPerPod - 1) / capacityPerPod
if targetReplicas <= target.active {
targetReplicas = target.active + 1
}
// Scale one replica at a time so a large batch cannot stampede the node.
replicas := int32(minInt(targetReplicas, target.active+1))
if err := s.deployments.Scale(ctx, target.namespace, target.name, replicas); err != nil {
return false, err
}
if s.events != nil {
if err := s.events.Publish(ctx, "runtime.pool.scaleout", map[string]any{
"runtime_type": runtimeType,
"namespace": target.namespace,
"deployment": target.name,
"replicas": replicas,
"reason": "pending_backlog",
"backlog": backlog,
}); err != nil {
log.Printf("runtime scheduler publish backlog scale-out event failed: %v", err)
}
}
return true, nil
}
func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error {
if s == nil || s.podRepo == nil {
return fmt.Errorf("runtime scheduler pod repository is not configured")
@@ -657,15 +818,31 @@ func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.I
if pod.AgentEndpoint == nil || strings.TrimSpace(*pod.AgentEndpoint) == "" {
continue
}
unlock := s.lockGatewayCreateForPod(pod.ID)
canStart, err := s.podCanStartGateway(ctx, pod.ID)
if err != nil {
unlock()
lastErr = err
continue
}
if !canStart {
unlock()
lastErr = fmt.Errorf("runtime pod %d has gateway start in flight: %w", pod.ID, errRuntimeGatewayStartPending)
continue
}
claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID)
if err != nil {
unlock()
lastErr = err
continue
}
if !claimed {
unlock()
continue
}
if err := s.createGatewayOnPod(ctx, instance, runtimeType, pod); err != nil {
err = s.createGatewayOnPod(ctx, instance, runtimeType, pod)
unlock()
if err != nil {
if releaseErr := s.podRepo.ReleaseSlot(ctx, pod.ID); releaseErr != nil {
return errors.Join(err, releaseErr)
}
@@ -681,10 +858,12 @@ func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.I
}
type runtimeDeploymentCapacity struct {
namespace string
name string
active int
full int
namespace string
name string
active int
full int
capacityTotal int
usedSlots int
}
func (s *RuntimeScheduler) scaleOutIfAtCapacity(ctx context.Context, runtimeType string) (bool, error) {
@@ -781,7 +960,7 @@ func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance mode
}
now := time.Now().UTC()
gatewayState := normalizeRuntimeGatewayCreateState(resp.Status)
lifecycle := NormalizeRuntimeGatewayLifecycle(resp.Status, nil)
binding := &models.InstanceRuntimeBinding{
InstanceID: instance.ID,
RuntimePodID: pod.ID,
@@ -790,41 +969,39 @@ func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance mode
GatewayPort: resp.Port,
GatewayPID: resp.PID,
WorkspacePath: workspacePath,
State: gatewayState,
State: lifecycle.BindingState,
Generation: instance.RuntimeGeneration,
}
if gatewayState == "running" {
if lifecycle.Running {
binding.LastHealthAt = &now
}
if err := s.bindingRepo.Create(ctx, binding); err != nil {
if err := s.createRuntimeBindingWithStalePortRecovery(ctx, binding); err != nil {
return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, false, err)
}
if !lifecycle.CreateAccepted() {
cause := fmt.Errorf("runtime gateway %s returned status %q", resp.GatewayID, strings.TrimSpace(resp.Status))
message := cause.Error()
var errs []error
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &message); err != nil {
errs = append(errs, fmt.Errorf("mark instance %d error: %w", instance.ID, err))
}
errs = append(errs, s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, cause))
return errors.Join(errs...)
}
if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err)
}
instanceState := "creating"
var stateMessage *string
if gatewayState == "running" {
instanceState = "running"
} else {
message := "runtime gateway starting"
stateMessage = &message
}
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, instanceState, instance.RuntimeGeneration, stateMessage); err != nil {
if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, lifecycle.InstanceState, instance.RuntimeGeneration, lifecycle.Message); err != nil {
return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err)
}
if s.events != nil {
eventType := "runtime.instance.starting"
if gatewayState == "running" {
eventType = "runtime.instance.running"
}
if err := s.events.Publish(ctx, eventType, map[string]any{
if err := s.events.Publish(ctx, lifecycle.EventType, map[string]any{
"instance_id": instance.ID,
"runtime_type": runtimeType,
"runtime_pod_id": pod.ID,
"gateway_id": resp.GatewayID,
"gateway_port": resp.Port,
"gateway_state": gatewayState,
"gateway_state": lifecycle.BindingState,
"workspace_path": workspacePath,
"generation": instance.RuntimeGeneration,
}); err != nil {
@@ -834,15 +1011,21 @@ func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance mode
return nil
}
func normalizeRuntimeGatewayCreateState(status string) string {
switch strings.ToLower(strings.TrimSpace(status)) {
case "running", "ready", "healthy":
return "running"
default:
return "starting"
func (s *RuntimeScheduler) createRuntimeBindingWithStalePortRecovery(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
if err := s.bindingRepo.Create(ctx, binding); err != nil {
deleted, deleteErr := s.bindingRepo.DeleteErrorByRuntimePodIDAndGatewayPort(ctx, binding.RuntimePodID, binding.GatewayPort)
if deleteErr != nil {
return errors.Join(err, deleteErr)
}
if deleted == 0 {
return err
}
if retryErr := s.bindingRepo.Create(ctx, binding); retryErr != nil {
return errors.Join(err, retryErr)
}
}
return nil
}
func runtimeGatewayLinuxIDs(instanceID int, environment map[string]string) (int, int) {
linuxID := RuntimeLinuxID(instanceID)
if !strings.EqualFold(strings.TrimSpace(environment["CLAWMANAGER_TEAM_ENABLED"]), "true") {
@@ -930,7 +1113,8 @@ func isRecoverableRuntimeSchedulingError(instance models.Instance) bool {
return false
}
message := strings.TrimSpace(*instance.RuntimeErrorMessage)
return message == fmt.Sprintf("no schedulable %s runtime pod", runtimeType)
return message == fmt.Sprintf("no schedulable %s runtime pod", runtimeType) ||
strings.Contains(message, fmt.Sprintf("no schedulable %s runtime pod:", runtimeType))
}
func minInt(a, b int) int {
@@ -3,7 +3,10 @@ package services
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -83,7 +86,7 @@ func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) {
if req.req.WorkspacePath != "/workspaces/openclaw/user-45/instance-17" {
t.Fatalf("workspace path = %q", req.req.WorkspacePath)
}
if req.req.PortRange.Start != 20000 || req.req.PortRange.End != 20099 {
if req.req.PortRange.Start != RuntimeGatewayPortStart || req.req.PortRange.End != RuntimeGatewayPortEnd {
t.Fatalf("port range = %+v", req.req.PortRange)
}
if req.req.UID != RuntimeLinuxID(17) || req.req.GID != RuntimeLinuxID(17) {
@@ -118,6 +121,359 @@ func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) {
}
}
func TestRuntimeSchedulerStoresStartingGatewayBindingAsCreating(t *testing.T) {
ctx := context.Background()
endpoint := "http://agent.runtime"
workspacePath := "/workspaces/openclaw/user-45/instance-72"
instanceRepo := newFakeRuntimeInstanceRepo()
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
9: {
ID: 9,
RuntimeType: RuntimeTypeOpenClaw,
AgentEndpoint: &endpoint,
State: "ready",
Capacity: 100,
},
},
schedulable: []models.RuntimePod{{
ID: 9,
RuntimeType: RuntimeTypeOpenClaw,
AgentEndpoint: &endpoint,
State: "ready",
Capacity: 100,
}},
}
bindingRepo := newFakeRuntimeBindingRepo()
agent := &fakeRuntimeAgentClient{
createResponse: &RuntimeAgentCreateGatewayResponse{
GatewayID: "gw-72",
Port: 20072,
Status: "starting",
},
}
scheduler := NewRuntimeScheduler(
instanceRepo,
podRepo,
bindingRepo,
&fakeRuntimeRolloutRepo{},
agent,
&fakeRuntimeEventService{},
nil,
&fakeRuntimeDeploymentService{},
time.Second,
)
err := scheduler.assignInstance(ctx, models.Instance{
ID: 72,
UserID: 45,
Type: RuntimeTypeOpenClaw,
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
Status: "creating",
WorkspacePath: &workspacePath,
RuntimeGeneration: 1,
})
if err != nil {
t.Fatalf("assignInstance returned error: %v", err)
}
binding := bindingRepo.bindings[72]
if binding == nil {
t.Fatal("binding was not created")
}
if binding.State != "creating" {
t.Fatalf("binding state = %q, want creating", binding.State)
}
if binding.LastHealthAt != nil {
t.Fatalf("last health at = %v, want nil until gateway is running", binding.LastHealthAt)
}
}
func TestRuntimeSchedulerMarksInstanceErrorWhenGatewayCreateReturnsErrorStatus(t *testing.T) {
ctx := context.Background()
endpoint := "http://agent.runtime"
workspacePath := "/workspaces/openclaw/user-45/instance-73"
instanceRepo := newFakeRuntimeInstanceRepo()
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
9: {
ID: 9,
RuntimeType: RuntimeTypeOpenClaw,
AgentEndpoint: &endpoint,
State: "ready",
Capacity: 100,
},
},
schedulable: []models.RuntimePod{{
ID: 9,
RuntimeType: RuntimeTypeOpenClaw,
AgentEndpoint: &endpoint,
State: "ready",
Capacity: 100,
}},
}
bindingRepo := newFakeRuntimeBindingRepo()
agent := &fakeRuntimeAgentClient{
createResponse: &RuntimeAgentCreateGatewayResponse{
GatewayID: "gw-73",
Port: 20073,
Status: "error",
},
}
events := &fakeRuntimeEventService{}
scheduler := NewRuntimeScheduler(
instanceRepo,
podRepo,
bindingRepo,
&fakeRuntimeRolloutRepo{},
agent,
events,
nil,
&fakeRuntimeDeploymentService{},
time.Second,
)
err := scheduler.assignInstance(ctx, models.Instance{
ID: 73,
UserID: 45,
Type: RuntimeTypeOpenClaw,
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
Status: "creating",
WorkspacePath: &workspacePath,
RuntimeGeneration: 2,
})
if err == nil {
t.Fatal("assignInstance returned nil error")
}
if !strings.Contains(err.Error(), `runtime gateway gw-73 returned status "error"`) {
t.Fatalf("assignInstance error = %v, want gateway status error", err)
}
state := instanceRepo.runtimeStates[73]
if state.status != "error" || state.generation != 2 || state.message == nil || !strings.Contains(*state.message, `runtime gateway gw-73 returned status "error"`) {
t.Fatalf("runtime state = %+v, want error with gateway status message", state)
}
if _, ok := instanceRepo.workspacePaths[73]; ok {
t.Fatalf("workspace path was set after gateway error: %q", instanceRepo.workspacePaths[73])
}
if bindingRepo.bindings[73] != nil {
t.Fatalf("binding remains after gateway error: %+v", bindingRepo.bindings[73])
}
if bindingRepo.deleteCalls[73] != 1 {
t.Fatalf("binding delete calls = %d, want 1", bindingRepo.deleteCalls[73])
}
if got := len(agent.deleteRequests); got != 1 {
t.Fatalf("DeleteGateway calls = %d, want 1", got)
}
if agent.deleteRequests[0].gatewayID != "gw-73" {
t.Fatalf("deleted gateway = %q, want gw-73", agent.deleteRequests[0].gatewayID)
}
if podRepo.releases[9] != 1 {
t.Fatalf("pod releases = %d, want 1", podRepo.releases[9])
}
if got := len(events.published); got != 0 {
t.Fatalf("published events = %d, want 0", got)
}
}
func TestRuntimeSchedulerThrottlesGatewayStartsWhenPodHasCreatingBinding(t *testing.T) {
ctx := context.Background()
endpoint := "http://agent.runtime"
workspacePath := "/workspaces/openclaw/user-45/instance-83"
pod := models.RuntimePod{
ID: 9,
RuntimeType: RuntimeTypeOpenClaw,
AgentEndpoint: &endpoint,
State: "ready",
Capacity: 100,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{9: &pod},
schedulable: []models.RuntimePod{pod},
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[82] = &models.InstanceRuntimeBinding{
InstanceID: 82,
RuntimePodID: 9,
RuntimeType: RuntimeTypeOpenClaw,
State: "creating",
Generation: 1,
}
agent := &fakeRuntimeAgentClient{
createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-83", Port: 20083, Status: "running"},
}
scheduler := NewRuntimeScheduler(
newFakeRuntimeInstanceRepo(),
podRepo,
bindingRepo,
&fakeRuntimeRolloutRepo{},
agent,
&fakeRuntimeEventService{},
nil,
&fakeRuntimeDeploymentService{},
time.Second,
)
err := scheduler.assignInstance(ctx, models.Instance{
ID: 83,
UserID: 45,
Type: RuntimeTypeOpenClaw,
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
Status: "creating",
WorkspacePath: &workspacePath,
RuntimeGeneration: 1,
})
if err == nil {
t.Fatal("assignInstance returned nil error")
}
if !strings.Contains(err.Error(), "gateway start in flight") {
t.Fatalf("assignInstance error = %v, want gateway start in flight", err)
}
if got := len(agent.createRequests); got != 0 {
t.Fatalf("CreateGateway calls = %d, want 0", got)
}
if podRepo.claims[9] != 0 {
t.Fatalf("pod claims = %d, want 0", podRepo.claims[9])
}
}
func TestRuntimeSchedulerReconcileWaitsWhenGatewayStartInFlight(t *testing.T) {
ctx := context.Background()
endpoint := "http://agent.runtime"
workspacePath := "/workspaces/openclaw/user-45/instance-84"
instanceRepo := newFakeRuntimeInstanceRepo()
instanceRepo.creating = []models.Instance{{
ID: 84,
UserID: 45,
Type: RuntimeTypeOpenClaw,
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
Status: "creating",
WorkspacePath: &workspacePath,
RuntimeGeneration: 1,
}}
pod := models.RuntimePod{
ID: 9,
RuntimeType: RuntimeTypeOpenClaw,
AgentEndpoint: &endpoint,
State: "ready",
Capacity: 100,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{9: &pod},
schedulable: []models.RuntimePod{pod},
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[82] = &models.InstanceRuntimeBinding{
InstanceID: 82,
RuntimePodID: 9,
RuntimeType: RuntimeTypeOpenClaw,
State: "creating",
Generation: 1,
}
agent := &fakeRuntimeAgentClient{
createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-84", Port: 20084, Status: "running"},
}
scheduler := NewRuntimeScheduler(
instanceRepo,
podRepo,
bindingRepo,
&fakeRuntimeRolloutRepo{},
agent,
&fakeRuntimeEventService{},
nil,
&fakeRuntimeDeploymentService{},
time.Second,
)
if err := scheduler.reconcile(ctx); err != nil {
t.Fatalf("reconcile returned error: %v", err)
}
if got := len(agent.createRequests); got != 0 {
t.Fatalf("CreateGateway calls = %d, want 0", got)
}
if podRepo.claims[9] != 0 {
t.Fatalf("pod claims = %d, want 0", podRepo.claims[9])
}
if state, ok := instanceRepo.runtimeStates[84]; ok {
t.Fatalf("runtime state was updated to %+v, want waiting without error", state)
}
}
func TestRuntimeSchedulerSerializesGatewayCreationPerPod(t *testing.T) {
ctx := context.Background()
endpoint := "http://agent.runtime"
workspace81 := "/workspaces/openclaw/user-45/instance-81"
workspace82 := "/workspaces/openclaw/user-45/instance-82"
instances := []models.Instance{
{
ID: 81,
UserID: 45,
Type: RuntimeTypeOpenClaw,
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
Status: "creating",
WorkspacePath: &workspace81,
RuntimeGeneration: 1,
},
{
ID: 82,
UserID: 45,
Type: RuntimeTypeOpenClaw,
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
Status: "creating",
WorkspacePath: &workspace82,
RuntimeGeneration: 1,
},
}
pod := models.RuntimePod{
ID: 9,
RuntimeType: RuntimeTypeOpenClaw,
AgentEndpoint: &endpoint,
State: "ready",
Capacity: 100,
}
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{9: &pod},
schedulable: []models.RuntimePod{pod},
}
bindingRepo := newFakeRuntimeBindingRepo()
agent := &fakeRuntimeAgentClient{
createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw", Port: 20081, Status: "running"},
createDelay: 25 * time.Millisecond,
}
scheduler := NewRuntimeScheduler(
newFakeRuntimeInstanceRepo(),
podRepo,
bindingRepo,
&fakeRuntimeRolloutRepo{},
agent,
&fakeRuntimeEventService{},
nil,
&fakeRuntimeDeploymentService{},
time.Second,
)
errs := make(chan error, len(instances))
for _, instance := range instances {
instance := instance
go func() {
errs <- scheduler.assignInstance(ctx, instance)
}()
}
for range instances {
if err := <-errs; err != nil {
t.Fatalf("assignInstance returned error: %v", err)
}
}
if got := atomic.LoadInt32(&agent.maxActiveCreates); got != 1 {
t.Fatalf("concurrent CreateGateway calls = %d, want 1", got)
}
if got := len(agent.createRequests); got != 2 {
t.Fatalf("CreateGateway calls = %d, want 2", got)
}
}
func TestRuntimeSchedulerPassesGatewayEnvironmentToRuntimeAgent(t *testing.T) {
ctx := context.Background()
endpoint := "http://agent.runtime"
@@ -371,6 +727,62 @@ func TestRuntimeSchedulerScalesOutWhenAllReadyPodsAtCapacity(t *testing.T) {
}
}
func TestRuntimeSchedulerScalesOutWhenBacklogExceedsReadyCapacity(t *testing.T) {
ctx := context.Background()
endpoint1 := "http://pod-1.runtime"
endpoint2 := "http://pod-2.runtime"
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
5: {
ID: 5,
RuntimeType: RuntimeTypeOpenClaw,
Namespace: "clawmanager-system",
DeploymentName: "openclaw-runtime",
AgentEndpoint: &endpoint1,
State: "ready",
Capacity: 100,
UsedSlots: 99,
},
6: {
ID: 6,
RuntimeType: RuntimeTypeOpenClaw,
Namespace: "clawmanager-system",
DeploymentName: "openclaw-runtime",
AgentEndpoint: &endpoint2,
State: "ready",
Capacity: 100,
UsedSlots: 0,
},
},
}
deployments := &fakeRuntimeDeploymentService{}
scheduler := NewRuntimeScheduler(
newFakeRuntimeInstanceRepo(),
podRepo,
newFakeRuntimeBindingRepo(),
&fakeRuntimeRolloutRepo{},
&fakeRuntimeAgentClient{},
NewRuntimeEventService(nil),
nil,
deployments,
time.Second,
)
scaled, err := scheduler.scaleOutIfBacklogExceedsCapacity(ctx, RuntimeTypeOpenClaw, 120)
if err != nil {
t.Fatalf("scaleOutIfBacklogExceedsCapacity returned error: %v", err)
}
if !scaled {
t.Fatal("scaleOutIfBacklogExceedsCapacity returned scaled=false")
}
if got := len(deployments.scales); got != 1 {
t.Fatalf("deployment Scale calls = %d, want 1", got)
}
scale := deployments.scales[0]
if scale.namespace != "clawmanager-system" || scale.name != "openclaw-runtime" || scale.replicas != 3 {
t.Fatalf("deployment Scale call = %+v, want clawmanager-system/openclaw-runtime replicas 3", scale)
}
}
func TestRuntimeSchedulerDoesNotScaleOutWhenGatewayCreateFailsWithFreeSlot(t *testing.T) {
ctx := context.Background()
endpoint := "http://pod-1.runtime"
@@ -480,7 +892,7 @@ func TestRuntimeSchedulerReconcileRetriesRecoverableNoSchedulableError(t *testin
ctx := context.Background()
endpoint := "http://agent.runtime"
workspacePath := "/workspaces/openclaw/user-46/instance-68"
errorMessage := "no schedulable openclaw runtime pod"
errorMessage := "no schedulable openclaw runtime pod: runtime agent conflict: no free port"
instanceRepo := newFakeRuntimeInstanceRepo()
instanceRepo.desiredRunning = []models.Instance{{
ID: 68,
@@ -850,6 +1262,101 @@ func TestRuntimeSchedulerCleansUpGatewayAndReleasesSlotWhenBindingCreateFails(t
}
}
func TestRuntimeSchedulerClearsStaleErrorBindingAndRetriesBindingCreate(t *testing.T) {
ctx := context.Background()
endpoint := "http://agent.runtime"
instanceRepo := newFakeRuntimeInstanceRepo()
podRepo := &fakeRuntimePodRepo{
pods: map[int64]*models.RuntimePod{
9: {ID: 9, RuntimeType: RuntimeTypeHermes, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
},
schedulable: []models.RuntimePod{
{ID: 9, RuntimeType: RuntimeTypeHermes, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
},
}
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.enforceUniqueGateway = true
bindingRepo.bindings[141] = &models.InstanceRuntimeBinding{
InstanceID: 141,
RuntimePodID: 9,
RuntimeType: RuntimeTypeHermes,
GatewayID: "gw-141-15",
GatewayPort: 20000,
State: "error",
Generation: 15,
}
agent := &fakeRuntimeAgentClient{
createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-218-5", Port: 20000, Status: "running"},
}
scheduler := NewRuntimeScheduler(
instanceRepo,
podRepo,
bindingRepo,
&fakeRuntimeRolloutRepo{},
agent,
NewRuntimeEventService(nil),
nil,
&fakeRuntimeDeploymentService{},
time.Second,
)
err := scheduler.assignInstance(ctx, models.Instance{
ID: 218,
UserID: 1,
Type: RuntimeTypeHermes,
RuntimeType: RuntimeBackendGateway,
InstanceMode: InstanceModeLite,
WorkspacePath: ptrString("/workspaces/hermes/user-1/instance-218"),
MemoryGB: 1,
DiskGB: 1,
RuntimeGeneration: 5,
})
if err != nil {
t.Fatalf("assignInstance returned error: %v", err)
}
if bindingRepo.bindings[141] != nil {
t.Fatal("stale error binding still blocks the gateway port")
}
binding := bindingRepo.bindings[218]
if binding == nil {
t.Fatal("new binding was not created after stale cleanup")
}
if binding.RuntimePodID != 9 || binding.GatewayPort != 20000 || binding.State != "running" {
t.Fatalf("new binding = %+v, want pod 9 port 20000 running", binding)
}
if got := bindingRepo.deleteByPodPortCalls["9/20000"]; got != 1 {
t.Fatalf("stale pod/port cleanup calls = %d, want 1", got)
}
if got := len(agent.deleteRequests); got != 0 {
t.Fatalf("DeleteGateway calls = %d, want 0", got)
}
if podRepo.releases[9] != 0 {
t.Fatalf("pod releases = %d, want 0", podRepo.releases[9])
}
}
func TestRuntimeSchedulerDoesNotClearStartingBindingOnGatewayPortConflict(t *testing.T) {
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[218] = &models.InstanceRuntimeBinding{
InstanceID: 218,
RuntimePodID: 9,
RuntimeType: RuntimeTypeHermes,
GatewayID: "gw-218-7",
GatewayPort: 20000,
State: "starting",
Generation: 7,
}
deleted, err := bindingRepo.DeleteErrorByRuntimePodIDAndGatewayPort(context.Background(), 9, 20000)
if err != nil {
t.Fatalf("DeleteErrorByRuntimePodIDAndGatewayPort returned error: %v", err)
}
if deleted != 0 {
t.Fatalf("deleted bindings = %d, want 0", deleted)
}
if bindingRepo.bindings[218] == nil || bindingRepo.bindings[218].State != "starting" {
t.Fatalf("starting binding was removed or changed: %+v", bindingRepo.bindings[218])
}
}
func TestRuntimeSchedulerCleansUpGatewayBindingAndSlotWhenWorkspacePathUpdateFails(t *testing.T) {
ctx := context.Background()
endpoint := "http://agent.runtime"
@@ -1563,6 +2070,7 @@ func (r *fakeRuntimeInstanceRepo) Update(instance *models.Instance) error { retu
func (r *fakeRuntimeInstanceRepo) Delete(id int) error { return nil }
type fakeRuntimePodRepo struct {
mu sync.Mutex
pods map[int64]*models.RuntimePod
schedulable []models.RuntimePod
claims map[int64]int
@@ -1602,6 +2110,8 @@ func (r *fakeRuntimePodRepo) ListSchedulable(ctx context.Context, runtimeType st
return r.schedulable, nil
}
func (r *fakeRuntimePodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
r.mu.Lock()
defer r.mu.Unlock()
if r.claims == nil {
r.claims = map[int64]int{}
}
@@ -1643,12 +2153,15 @@ func (r *fakeRuntimePodRepo) UpdateMetrics(ctx context.Context, podID int64, met
}
type fakeRuntimeBindingRepo struct {
mu sync.Mutex
bindings map[int]*models.InstanceRuntimeBinding
deleteCalls map[int]int
deleteAndReleaseCalls map[int]int
runningErr error
getErr error
createErr error
enforceUniqueGateway bool
deleteByPodPortCalls map[string]int
}
func newFakeRuntimeBindingRepo() *fakeRuntimeBindingRepo {
@@ -1656,24 +2169,38 @@ func newFakeRuntimeBindingRepo() *fakeRuntimeBindingRepo {
bindings: map[int]*models.InstanceRuntimeBinding{},
deleteCalls: map[int]int{},
deleteAndReleaseCalls: map[int]int{},
deleteByPodPortCalls: map[string]int{},
}
}
func (r *fakeRuntimeBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
r.mu.Lock()
defer r.mu.Unlock()
if r.createErr != nil {
return r.createErr
}
if r.enforceUniqueGateway {
for _, existing := range r.bindings {
if existing != nil && existing.InstanceID != binding.InstanceID && existing.RuntimePodID == binding.RuntimePodID && existing.GatewayPort == binding.GatewayPort {
return fmt.Errorf("failed to create instance runtime binding: duplicate gateway")
}
}
}
copy := *binding
r.bindings[binding.InstanceID] = &copy
return nil
}
func (r *fakeRuntimeBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
r.mu.Lock()
defer r.mu.Unlock()
if r.getErr != nil {
return nil, r.getErr
}
return r.bindings[instanceID], nil
}
func (r *fakeRuntimeBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
r.mu.Lock()
defer r.mu.Unlock()
if r.runningErr != nil {
return nil, r.runningErr
}
@@ -1684,6 +2211,8 @@ func (r *fakeRuntimeBindingRepo) GetRunningByInstanceID(ctx context.Context, ins
return binding, nil
}
func (r *fakeRuntimeBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
r.mu.Lock()
defer r.mu.Unlock()
var bindings []models.InstanceRuntimeBinding
for _, binding := range r.bindings {
if binding.RuntimePodID == runtimePodID {
@@ -1701,6 +2230,21 @@ func (r *fakeRuntimeBindingRepo) UpdateRunning(ctx context.Context, instanceID i
func (r *fakeRuntimeBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
return nil
}
func (r *fakeRuntimeBindingRepo) DeleteErrorByRuntimePodIDAndGatewayPort(ctx context.Context, runtimePodID int64, gatewayPort int) (int64, error) {
r.mu.Lock()
defer r.mu.Unlock()
key := fmt.Sprintf("%d/%d", runtimePodID, gatewayPort)
r.deleteByPodPortCalls[key]++
var deleted int64
for instanceID, binding := range r.bindings {
if binding == nil || binding.RuntimePodID != runtimePodID || binding.GatewayPort != gatewayPort || binding.State != "error" {
continue
}
delete(r.bindings, instanceID)
deleted++
}
return deleted, nil
}
func (r *fakeRuntimeBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
r.deleteCalls[instanceID]++
delete(r.bindings, instanceID)
@@ -1711,6 +2255,15 @@ func (r *fakeRuntimeBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Co
delete(r.bindings, instanceID)
return nil
}
func (r *fakeRuntimeBindingRepo) DeleteRunningByInstanceIDGenerationAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64, generation int) (bool, error) {
binding := r.bindings[instanceID]
if binding == nil || binding.RuntimePodID != runtimePodID || binding.Generation != generation || binding.State != "running" {
return false, nil
}
r.deleteAndReleaseCalls[instanceID]++
delete(r.bindings, instanceID)
return true, nil
}
type fakeRuntimeRolloutRepo struct {
rollouts map[int64]*models.RuntimeRollout
@@ -1755,12 +2308,17 @@ func (r *fakeRuntimeRolloutRepo) UpdateStatus(ctx context.Context, id int64, sta
}
type fakeRuntimeAgentClient struct {
mu sync.Mutex
createResponse *RuntimeAgentCreateGatewayResponse
createErr error
deleteErr error
createDelay time.Duration
createRequests []fakeCreateGatewayRequest
deleteRequests []fakeDeleteGatewayRequest
drainEndpoints []string
activeCreates int32
maxActiveCreates int32
}
type fakeCreateGatewayRequest struct {
@@ -1775,6 +2333,23 @@ type fakeDeleteGatewayRequest struct {
func (c *fakeRuntimeAgentClient) Health(ctx context.Context, endpoint string) error { return nil }
func (c *fakeRuntimeAgentClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) {
active := atomic.AddInt32(&c.activeCreates, 1)
defer atomic.AddInt32(&c.activeCreates, -1)
for {
current := atomic.LoadInt32(&c.maxActiveCreates)
if active <= current || atomic.CompareAndSwapInt32(&c.maxActiveCreates, current, active) {
break
}
}
if c.createDelay > 0 {
select {
case <-time.After(c.createDelay):
case <-ctx.Done():
return nil, ctx.Err()
}
}
c.mu.Lock()
defer c.mu.Unlock()
c.createRequests = append(c.createRequests, fakeCreateGatewayRequest{endpoint: endpoint, req: req})
if c.createErr != nil {
return nil, c.createErr
+6 -4
View File
@@ -6193,6 +6193,8 @@ spec:
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: TEAM_REDIS_URL
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: CLAWMANAGER_DESKTOP_DIRECT_PROXY
value: "true"
- name: CLAWMANAGER_TEAM_REDIS_URL
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: CLAWMANAGER_TEAM_MANAGER_BASE_URL
@@ -6208,7 +6210,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_SCHEDULER_ENABLED
value: "true"
- name: RUNTIME_HEARTBEAT_TIMEOUT
@@ -6305,7 +6307,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_AGENT_LISTEN_ADDR
value: "0.0.0.0:19090"
- name: RUNTIME_AGENT_PUBLIC_PORT
@@ -6392,7 +6394,7 @@ spec:
- name: CLAWMANAGER_GATEWAY_PORT_START
value: "20000"
- name: CLAWMANAGER_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: HERMES_TUI_DIR
value: /usr/local/lib/hermes-agent/ui-tui
- name: CLAWMANAGER_AGENT_CONTROL_TOKEN
@@ -6414,7 +6416,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_AGENT_CONTROL_TOKEN
valueFrom:
secretKeyRef:
+4 -4
View File
@@ -1368,7 +1368,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_SCHEDULER_ENABLED
value: "true"
- name: RUNTIME_HEARTBEAT_TIMEOUT
@@ -1465,7 +1465,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_AGENT_LISTEN_ADDR
value: "0.0.0.0:19090"
- name: RUNTIME_AGENT_PUBLIC_PORT
@@ -1552,7 +1552,7 @@ spec:
- name: CLAWMANAGER_GATEWAY_PORT_START
value: "20000"
- name: CLAWMANAGER_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: HERMES_TUI_DIR
value: /usr/local/lib/hermes-agent/ui-tui
- name: CLAWMANAGER_AGENT_CONTROL_TOKEN
@@ -1574,7 +1574,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_AGENT_CONTROL_TOKEN
valueFrom:
secretKeyRef:
+6 -4
View File
@@ -6193,6 +6193,8 @@ spec:
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: TEAM_REDIS_URL
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: CLAWMANAGER_DESKTOP_DIRECT_PROXY
value: "true"
- name: CLAWMANAGER_TEAM_REDIS_URL
value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: CLAWMANAGER_TEAM_MANAGER_BASE_URL
@@ -6208,7 +6210,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_SCHEDULER_ENABLED
value: "true"
- name: RUNTIME_HEARTBEAT_TIMEOUT
@@ -6305,7 +6307,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_AGENT_LISTEN_ADDR
value: "0.0.0.0:19090"
- name: RUNTIME_AGENT_PUBLIC_PORT
@@ -6392,7 +6394,7 @@ spec:
- name: CLAWMANAGER_GATEWAY_PORT_START
value: "20000"
- name: CLAWMANAGER_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: HERMES_TUI_DIR
value: /usr/local/lib/hermes-agent/ui-tui
- name: CLAWMANAGER_AGENT_CONTROL_TOKEN
@@ -6414,7 +6416,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_AGENT_CONTROL_TOKEN
valueFrom:
secretKeyRef:
+4 -4
View File
@@ -1368,7 +1368,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_SCHEDULER_ENABLED
value: "true"
- name: RUNTIME_HEARTBEAT_TIMEOUT
@@ -1465,7 +1465,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_AGENT_LISTEN_ADDR
value: "0.0.0.0:19090"
- name: RUNTIME_AGENT_PUBLIC_PORT
@@ -1552,7 +1552,7 @@ spec:
- name: CLAWMANAGER_GATEWAY_PORT_START
value: "20000"
- name: CLAWMANAGER_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: HERMES_TUI_DIR
value: /usr/local/lib/hermes-agent/ui-tui
- name: CLAWMANAGER_AGENT_CONTROL_TOKEN
@@ -1574,7 +1574,7 @@ spec:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_AGENT_CONTROL_TOKEN
valueFrom:
secretKeyRef:
+86
View File
@@ -37,3 +37,89 @@ End users use ClawManager to:
2. Review Agent Control Plane if your focus is runtime visibility and operations.
3. Review Resource Management and Security Center if you want reusable channels, skills, and scan-backed workflows.
4. Create a Team from a role template when a task benefits from coordinated planning, delivery, and review.
## Desktop Clipboard
Desktop-mode instances based on WebTop, including OpenClaw Pro and Hermes, use Selkies for browser-to-desktop clipboard synchronization. New instances enable clipboard synchronization in both directions by default.
| Direction | Default behavior |
| --- | --- |
| Local browser / host to WebTop | Enabled. Copy content locally, focus an application inside WebTop, then use `Ctrl+V` in that application. |
| WebTop to local browser / host | Enabled. Copy content inside WebTop to make it available locally. |
Copying locally only updates the WebTop clipboard. It does not automatically place content into a chat box, terminal, or editor: first focus the target application inside the remote desktop, then paste there.
### Change clipboard behavior for one new instance
In the instance creation page, use **Custom variables**. Values must be entered as plain text: do not include Markdown backticks, quotes, or spaces.
#### Disable clipboard synchronization in both directions
Use this when Unicode/IME input occasionally inserts old clipboard content, particularly under a busy desktop or browser workload.
| Variable | Value |
| --- | --- |
| `SELKIES_CLIPBOARD_ENABLED` | `false|locked` |
| `SELKIES_CLIPBOARD_IN_ENABLED` | `false|locked` |
| `SELKIES_CLIPBOARD_OUT_ENABLED` | `false|locked` |
#### Allow only local browser / host to WebTop
Use this when users need to paste external content into WebTop but must not copy data from WebTop back to the local machine.
| Variable | Value |
| --- | --- |
| `SELKIES_CLIPBOARD_ENABLED` | `true|locked` |
| `SELKIES_CLIPBOARD_IN_ENABLED` | `true|locked` |
| `SELKIES_CLIPBOARD_OUT_ENABLED` | `false|locked` |
`SELKIES_CLIPBOARD_ENABLED` is a boolean setting. Do not set it to `in|locked` or `out|locked`: Selkies interprets those as disabled. The `in` and `out` modes are calculated internally from the three boolean settings above.
For legacy Kasm-compatible images, also set `KASM_SVC_SEND_CUT_TEXT=-SendCutText 0` and `KASM_SVC_ACCEPT_CUT_TEXT=-AcceptCutText 0` when disabling all clipboard synchronization. Current Selkies WebSocket desktops rely on the three `SELKIES_CLIPBOARD_*` settings.
### When changes take effect
Clipboard environment variables are read when the desktop Pod starts. They affect only the new instance created with those variables. To change an existing instance, first update its Deployment environment through the control plane or Kubernetes, then restart its Pod; alternatively, create a new instance with the desired custom variables. Changing an environment variable in a terminal inside a running WebTop session has no effect.
### Verify the effective configuration
Inside the WebTop terminal, run:
```bash
env | grep SELKIES_CLIPBOARD
```
For bidirectional synchronization, the output must be exactly:
```text
SELKIES_CLIPBOARD_ENABLED=true|locked
SELKIES_CLIPBOARD_IN_ENABLED=true|locked
SELKIES_CLIPBOARD_OUT_ENABLED=true|locked
```
Runtime logs provide the authoritative confirmation. A healthy inbound-only configuration reports:
```text
clipboard_enabled: (True, True)
clipboard_in_enabled: (True, True)
clipboard_out_enabled: (False, True)
```
When content from the local browser is copied into WebTop, the runtime logs:
```text
Clipboard direction=inbound ... status=applied
```
If the logs instead show `clipboard_enabled: (False, ...)`, check for an invalid value. A common mistake is entering a leading backtick, such as `` `true`` rather than `true|locked`.
### IME and Unicode input troubleshooting
KWin/Wayland can use an internal clipboard fallback to inject Unicode characters. Disabling Selkies clipboard synchronization prevents local clipboard content from being synchronized into WebTop, but it does not remove that internal Unicode fallback. If a user reports that Chinese, Hangul, or another IME input sporadically becomes clipboard text:
1. Create a diagnostic instance with both clipboard directions disabled.
2. Keep a unique sentinel value in the local clipboard and repeatedly enter Unicode text in the affected WebTop application.
3. Confirm that the sentinel appears only after an explicit paste, never as part of ordinary typing.
4. Review the runtime logs for `Clipboard direction=inbound ... status=applied` while reproducing the issue.
For a permanent runtime-level solution, use a Unicode input path that does not temporarily write to the system clipboard. Clipboard environment overrides are an instance-level mitigation and should be preferred when the issue is limited to a particular user or workload.
+2 -2
View File
@@ -71,7 +71,7 @@ agent 至少支持以下环境变量:
| `CLAWMANAGER_RUNTIME_TYPE` | runtime 类型,例如 `openclaw``hermes`。 |
| `RUNTIME_WORKSPACE_ROOT` | workspace 根目录,默认 `/workspaces`。 |
| `RUNTIME_GATEWAY_PORT_START` | gateway 端口起始值,默认 `20000`。 |
| `RUNTIME_GATEWAY_PORT_END` | gateway 端口结束值,默认 `20099`。 |
| `RUNTIME_GATEWAY_PORT_END` | gateway 端口结束值,默认 `20299`。 |
| `RUNTIME_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用。 |
| `RUNTIME_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用。 |
| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址,推荐 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。 |
@@ -108,7 +108,7 @@ Content-Type: application/json
"workspace_path": "/workspaces/hermes/user-45/instance-123",
"port_range": {
"start": 20000,
"end": 20099
"end": 20299
},
"uid": 200123,
"gid": 200123,
+1 -1
View File
@@ -101,7 +101,7 @@ Request body:
"workspace_path": "/workspaces/openclaw/user-45/instance-123",
"port_range": {
"start": 20000,
"end": 20099
"end": 20299
},
"uid": 200123,
"gid": 200123,
@@ -29,7 +29,7 @@
| --- | --- |
| workspace root | `/workspaces` |
| workspace path | `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` |
| gateway 端口范围 | `20000-20099` |
| gateway 端口范围 | `20000-20299` |
| 单 Runtime Pod 容量 | `100` |
| 实例 UID/GID | `200000 + instance_id` |
| agent 控制端口 | `19090` |
@@ -338,7 +338,7 @@ POST /v1/gateways
"workspace_path": "/workspaces/openclaw/user-45/instance-123",
"port_range": {
"start": 20000,
"end": 20099
"end": 20299
},
"uid": 200123,
"gid": 200123,
+4 -4
View File
@@ -24,7 +24,7 @@
| ClawManager 调 agent | `GET /v1/health``POST /v1/gateways``DELETE /v1/gateways/{id}``POST /v1/drain` | 不适用 |
| agent 上报 ClawManager | `/api/v1/runtime-agent/*` | `/api/v1/agent/*` |
| 用户实例进程 | 每个 Lite 实例是一个 gateway 子进程 | 整个桌面容器就是实例 |
| 用户访问端口 | agent 分配 `20000-20099` 中的 gateway 端口 | Webtop/KasmVNC `3001` |
| 用户访问端口 | agent 分配 `20000-20299` 中的 gateway 端口 | Webtop/KasmVNC `3001` |
| workspace | `/workspaces/hermes/user-{user_id}/instance-{instance_id}` | `/config/.hermes` |
| 典型实现参考 | 按 OpenClaw Lite 的 Runtime Pod Agent V2 做 | 按当前 Hermes Webtop guide 做 |
@@ -56,7 +56,7 @@ Lite agent 是共享 Pod 内的控制进程,不是某个用户实例内部的
- 校验 `X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN}`
- 实现 `GET /v1/health``POST /v1/gateways``DELETE /v1/gateways/{gateway_id}``POST /v1/drain`
-`${CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/*` 注册、heartbeat、metrics、gateway 状态和 skill inventory。
- 管理本 Pod 内多个 Hermes gateway 子进程,端口默认从 `20000-20099` 分配。
- 管理本 Pod 内多个 Hermes gateway 子进程,端口默认从 `20000-20299` 分配。
-`instance_id + generation` 做幂等创建,重复请求不能启动多个进程。
- `POST /v1/gateways` 必须快速返回 `starting`,后台继续启动和健康检查。
- gateway 健康后再上报 `running`;缺少 LLM token、workspace 越权、端口冲突或 Hermes 配置失败时上报 `error`
@@ -96,7 +96,7 @@ Content-Type: application/json
"agent_type": "hermes",
"workspace_path": "/workspaces/hermes/user-45/instance-123",
"gateway_port_start": 20000,
"gateway_port_end": 20099,
"gateway_port_end": 20299,
"uid": 200123,
"gid": 200123,
"environment": {
@@ -217,7 +217,7 @@ Hermes Pro 验收:
| 错误 | 正确做法 |
| --- | --- |
| 只实现 Pro instance agent,然后期望 Lite 可用 | Lite 必须实现 Runtime Pod Agent V2 |
| Lite agent 监听 `3001` | Lite agent 监听 `19090`gateway 子进程监听 `20000-20099` |
| Lite agent 监听 `3001` | Lite agent 监听 `19090`gateway 子进程监听 `20000-20299` |
| Lite 所有实例共用 `/config/.hermes` | Lite 每个实例使用自己的 `/workspaces/hermes/user-{uid}/instance-{id}` |
| `POST /v1/gateways` 阻塞到 Hermes gateway 完全启动 | 快速返回 `starting`,后台健康检查后上报 `running` |
| 重试 create 时启动多个 gateway | 按 `instance_id + generation` 幂等 |
@@ -115,7 +115,7 @@ Gateway create request:
"user_id": 45,
"agent_type": "openclaw",
"workspace_path": "/workspaces/openclaw/user-45/instance-123",
"port_range": { "start": 20000, "end": 20099 },
"port_range": { "start": 20000, "end": 20299 },
"uid": 200123,
"gid": 200123,
"cpu_cores": 2,
@@ -136,7 +136,7 @@ Gateway create response:
}
```
The agent must allocate a free port inside `20000-20099`, enforce one Linux UID/GID per instance using `200000 + instance_id`, apply cgroup CPU/memory limits, enforce workspace quota, reject symlink escapes inside workspaces, report gateway health, and survive backend replica changes. During gray upgrade, `POST /v1/drain` must reject new gateways and continue reporting existing gateway health until ClawManager migrates them away.
The agent must allocate a free port inside `20000-20299`, enforce one Linux UID/GID per instance using `200000 + instance_id`, apply cgroup CPU/memory limits, enforce workspace quota, reject symlink escapes inside workspaces, report gateway health, and survive backend replica changes. During gray upgrade, `POST /v1/drain` must reject new gateways and continue reporting existing gateway health until ClawManager migrates them away.
## Task 1: Schema, Models, And Repository Interfaces
@@ -670,7 +670,7 @@ func TestRuntimeAgentClientCreateGateway(t *testing.T) {
UserID: 8,
AgentType: "openclaw",
WorkspacePath: "/workspaces/openclaw/user-8/instance-7",
PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099},
PortRange: RuntimeAgentPortRange{Start: 20000, End: 20299},
UID: 200007,
GID: 200007,
CPUCores: 2,
@@ -931,7 +931,7 @@ const (
RuntimeTypeHermes = "hermes"
RuntimeGatewayPortStart = 20000
RuntimeGatewayPortEnd = 20099
RuntimeGatewayPortEnd = 20299
RuntimePodCapacity = 100
RuntimeLinuxIDBase = 200000
)
@@ -1133,7 +1133,7 @@ func BuildRuntimeDeployment(spec RuntimeDeploymentSpec) *appsv1.Deployment {
{Name: "CLAWMANAGER_RUNTIME_TYPE", Value: spec.RuntimeType},
{Name: "CLAWMANAGER_AGENT_PORT", Value: "19090"},
{Name: "CLAWMANAGER_GATEWAY_PORT_START", Value: "20000"},
{Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: "20099"},
{Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: "20299"},
{Name: "CLAWMANAGER_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken},
{Name: "CLAWMANAGER_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken},
},
@@ -2867,7 +2867,7 @@ Add env:
- name: RUNTIME_GATEWAY_PORT_START
value: "20000"
- name: RUNTIME_GATEWAY_PORT_END
value: "20099"
value: "20299"
- name: RUNTIME_SCHEDULER_ENABLED
value: "true"
- name: RUNTIME_AGENT_CONTROL_TOKEN
@@ -2926,7 +2926,7 @@ spec:
- name: CLAWMANAGER_GATEWAY_PORT_START
value: "20000"
- name: CLAWMANAGER_GATEWAY_PORT_END
value: "20099"
value: "20299"
volumeMounts:
- name: workspaces
mountPath: /workspaces
@@ -3075,6 +3075,6 @@ Placeholder scan:
Type consistency:
- Runtime types are consistently `openclaw` and `hermes`.
- Runtime Pod capacity is consistently 100.
- Gateway port range is consistently `20000-20099`.
- Gateway port range is consistently `20000-20299`.
- Workspace paths are consistently `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}`.
- Runtime user and group IDs are consistently `200000 + instance_id`.
@@ -65,7 +65,7 @@ backend replicas: 3
openclaw runtime replicas: 1
hermes runtime replicas: 1
gateways per runtime pod: 100
gateway port range per pod: 20000-20099
gateway port range per pod: 20000-20299
scheduler lease renew: 2s
scheduler failover: 6-8s
agent heartbeat interval: 2s
@@ -337,7 +337,7 @@ Reporting expands from one instance to Pod plus many gateways:
"pod_ip": "10.42.1.83",
"capacity": 100,
"used": 37,
"port_range": { "start": 20000, "end": 20099 },
"port_range": { "start": 20000, "end": 20299 },
"version": "openclaw:2.1.0",
"gateways": [
{
@@ -386,7 +386,7 @@ Create gateway request:
"user_id": 7,
"agent_type": "openclaw",
"workspace_path": "/workspaces/openclaw/user-7/instance-42",
"port_range": { "start": 20000, "end": 20099 },
"port_range": { "start": 20000, "end": 20299 },
"env": {
"CLAWMANAGER_INSTANCE_ID": "42",
"CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm"
@@ -413,7 +413,7 @@ Create gateway response:
Agent port requirements:
- Pod-local gateway range is `20000-20099`.
- Pod-local gateway range is `20000-20299`.
- Agent assigns the actual port.
- Agent skips conflicts or returns structured errors.
- Agent persists enough state to recover the port occupation map after restart.
@@ -541,7 +541,6 @@ const InstanceListPage: React.FC = () => {
<input
type="number"
min={1}
max={20}
value={batchCreateCount}
onChange={(event) => setBatchCreateCount(Number(event.target.value))}
className="app-input mt-1 w-full"